feat(postgres): State Store claims serve ownership with a session advisory lock

This commit is contained in:
sam committed 2026-09-24 02:57:38 +10:00
1 parent 972355fc63
commit a71d062b4a
4 files changed
+176

No files matched your search

+13
View File
@@ -13,6 +13,9 @@ import (
// from starting rather than falling back to SQLite, and a migration failure is
// a startup failure, not a warning.
//
// It then claims serve ownership of the database for the process's life, so a
// second State Store against the same database refuses to start.
//
// The pool is stored on boot.pg and closed at shutdown; nothing opens a second
// pool per subsystem.
func (b *boot) openStateStorePool(ctx context.Context) error {
@@ -32,7 +35,17 @@ func (b *boot) openStateStorePool(ctx context.Context) error {
b.log.Error("migrate state store postgres schema", "err", err)
return err
}
ownership, err := postgres.AcquireOwnership(ctx, pool, "state-store")
if err != nil {
pool.Close()
b.log.Error("claim state store ownership", "err", err)
return err
}
b.pg = pool
b.addCleanup(pool.Close)
// Registered after the pool so it runs first: the claim is dropped before
// the pool closes.
releaseCtx := context.WithoutCancel(ctx)
b.addCleanup(func() { _ = ownership.Release(releaseCtx) })
return nil
}
@@ -1,10 +1,13 @@
package archied
import (
"errors"
"strings"
"testing"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/infrastructure/postgres"
"github.com/samcharles93/archie-core/internal/infrastructure/postgres/pgtest"
)
// TestOpenStateStorePoolFailsClosedWithoutURL pins D1.2: the State Store is
@@ -30,3 +33,29 @@ func TestOpenStateStorePoolFailsClosedOnInvalidURL(t *testing.T) {
t.Fatal("openStateStorePool with an unparseable database_url succeeded; want fail closed")
}
}
// TestOpenStateStorePoolRefusesASecondOwner pins the serve-ownership claim: a
// second State Store against the same database fails to start while the first
// holds it, and starts once the first has shut down.
func TestOpenStateStorePoolRefusesASecondOwner(t *testing.T) {
url := pgtest.URL(t)
first := newBootstrap()
first.cfg = config.Config{DatabaseURL: url}
if err := first.openStateStorePool(t.Context()); err != nil {
t.Fatalf("first openStateStorePool: %v", err)
}
second := newBootstrap()
second.cfg = config.Config{DatabaseURL: url}
err := second.openStateStorePool(t.Context())
second.cleanup()
first.cleanup()
if !errors.Is(err, postgres.ErrOwned) {
t.Fatalf("second openStateStorePool = %v, want ErrOwned", err)
}
third := newBootstrap()
third.cfg = config.Config{DatabaseURL: url}
if err := third.openStateStorePool(t.Context()); err != nil {
t.Fatalf("openStateStorePool after the owner shut down: %v", err)
}
third.cleanup()
}
@@ -0,0 +1,59 @@
package postgres
import (
"context"
"errors"
"fmt"
"github.com/jackc/pgx/v5/pgxpool"
)
// ErrOwned is returned when another process already owns the named role on
// this database.
var ErrOwned = errors.New("postgres: owned by another process")
// Ownership is one process's exclusive claim to serve a role (the State Store,
// the Gateway) against a database. It is a session-level advisory lock held on
// a connection taken out of the pool for the claim's whole life, so a process
// that crashes or loses its connection releases it without cleanup.
type Ownership struct {
conn *pgxpool.Conn
name string
pid uint32
}
// AcquireOwnership claims name without blocking. A claim another process holds
// comes back as ErrOwned, never as a wait: a second owner must fail to start,
// not queue behind the first.
func AcquireOwnership(ctx context.Context, pool *pgxpool.Pool, name string) (*Ownership, error) {
conn, err := pool.Acquire(ctx)
if err != nil {
return nil, fmt.Errorf("postgres: ownership %s: %w", name, err)
}
var taken bool
if err := conn.QueryRow(ctx, "SELECT pg_try_advisory_lock(hashtextextended('archie.ownership.' || $1, 0))", name).Scan(&taken); err != nil {
conn.Release()
return nil, fmt.Errorf("postgres: ownership %s: %w", name, err)
}
if !taken {
conn.Release()
return nil, fmt.Errorf("%s: %w", name, ErrOwned)
}
return &Ownership{conn: conn, name: name, pid: conn.Conn().PgConn().PID()}, nil
}
// Release drops the claim and returns the connection. A connection that
// cannot unlock is closed rather than pooled, which ends the session and with
// it the lock. Release is idempotent.
func (o *Ownership) Release(ctx context.Context) error {
if o == nil || o.conn == nil {
return nil
}
conn := o.conn
o.conn = nil
if _, err := conn.Exec(ctx, "SELECT pg_advisory_unlock(hashtextextended('archie.ownership.' || $1, 0))", o.name); err != nil {
return errors.Join(fmt.Errorf("postgres: release ownership %s: %w", o.name, err), conn.Hijack().Close(ctx))
}
conn.Release()
return nil
}
@@ -0,0 +1,75 @@
package postgres
import (
"errors"
"testing"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/samcharles93/archie-core/internal/infrastructure/postgres/pgtest"
)
func openPool(t *testing.T, url string) *pgxpool.Pool {
t.Helper()
pool, err := Open(t.Context(), url)
if err != nil {
t.Fatalf("Open: %v", err)
}
t.Cleanup(pool.Close)
return pool
}
// claimOther takes name as another process would, then optionally releases it
// or kills its backend.
func claimOther(t *testing.T, url, name string, release, terminate bool) {
t.Helper()
other, err := AcquireOwnership(t.Context(), openPool(t, url), name)
if err != nil {
t.Fatalf("first claim: %v", err)
}
t.Cleanup(func() { _ = other.Release(t.Context()) })
if release {
if err := other.Release(t.Context()); err != nil {
t.Fatalf("Release: %v", err)
}
}
if terminate {
if _, err := openPool(t, url).Exec(t.Context(), "SELECT pg_terminate_backend($1)", other.pid); err != nil {
t.Fatalf("terminate: %v", err)
}
}
}
func TestAcquireOwnership(t *testing.T) {
tests := []struct {
name string
// held is taken by another process before the claim under test.
held string
// release drops the other claim before the claim under test.
release bool
// terminate kills the other claim's backend, as a crash would.
terminate bool
want error
}{
{name: "state-store", want: nil},
{name: "state-store", held: "state-store", want: ErrOwned},
{name: "state-store", held: "state-store", release: true, want: nil},
{name: "state-store", held: "state-store", terminate: true, want: nil},
{name: "gateway", held: "state-store", want: nil},
}
for _, tt := range tests {
t.Run(tt.name+"/held="+tt.held, func(t *testing.T) {
url := pgtest.URL(t)
if tt.held != "" {
claimOther(t, url, tt.held, tt.release, tt.terminate)
}
got, err := AcquireOwnership(t.Context(), openPool(t, url), tt.name)
if got != nil {
t.Cleanup(func() { _ = got.Release(t.Context()) })
}
if !errors.Is(err, tt.want) {
t.Fatalf("AcquireOwnership(%q) = %v, want %v", tt.name, err, tt.want)
}
})
}
}