From a71d062b4ab6d24766ebae1bf54cc47a05ebc037 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Thu, 24 Sep 2026 02:57:38 +1000 Subject: [PATCH] feat(postgres): State Store claims serve ownership with a session advisory lock --- internal/app/archied/state_store_pool.go | 13 ++++ internal/app/archied/state_store_pool_test.go | 29 +++++++ internal/infrastructure/postgres/ownership.go | 59 +++++++++++++++ .../infrastructure/postgres/ownership_test.go | 75 +++++++++++++++++++ 4 files changed, 176 insertions(+) create mode 100644 internal/infrastructure/postgres/ownership.go create mode 100644 internal/infrastructure/postgres/ownership_test.go diff --git a/internal/app/archied/state_store_pool.go b/internal/app/archied/state_store_pool.go index 402b62c85..e54959173 100644 --- a/internal/app/archied/state_store_pool.go +++ b/internal/app/archied/state_store_pool.go @@ -13,6 +13,9 @@ import ( // from starting rather than falling back to SQLite, and a migration failure is // a startup failure, not a warning. // +// It then claims serve ownership of the database for the process's life, so a +// second State Store against the same database refuses to start. +// // The pool is stored on boot.pg and closed at shutdown; nothing opens a second // pool per subsystem. func (b *boot) openStateStorePool(ctx context.Context) error { @@ -32,7 +35,17 @@ func (b *boot) openStateStorePool(ctx context.Context) error { b.log.Error("migrate state store postgres schema", "err", err) return err } + ownership, err := postgres.AcquireOwnership(ctx, pool, "state-store") + if err != nil { + pool.Close() + b.log.Error("claim state store ownership", "err", err) + return err + } b.pg = pool b.addCleanup(pool.Close) + // Registered after the pool so it runs first: the claim is dropped before + // the pool closes. + releaseCtx := context.WithoutCancel(ctx) + b.addCleanup(func() { _ = ownership.Release(releaseCtx) }) return nil } diff --git a/internal/app/archied/state_store_pool_test.go b/internal/app/archied/state_store_pool_test.go index 84835468d..11c0f0b4c 100644 --- a/internal/app/archied/state_store_pool_test.go +++ b/internal/app/archied/state_store_pool_test.go @@ -1,10 +1,13 @@ package archied import ( + "errors" "strings" "testing" "github.com/samcharles93/archie-core/internal/config" + "github.com/samcharles93/archie-core/internal/infrastructure/postgres" + "github.com/samcharles93/archie-core/internal/infrastructure/postgres/pgtest" ) // TestOpenStateStorePoolFailsClosedWithoutURL pins D1.2: the State Store is @@ -30,3 +33,29 @@ func TestOpenStateStorePoolFailsClosedOnInvalidURL(t *testing.T) { t.Fatal("openStateStorePool with an unparseable database_url succeeded; want fail closed") } } + +// TestOpenStateStorePoolRefusesASecondOwner pins the serve-ownership claim: a +// second State Store against the same database fails to start while the first +// holds it, and starts once the first has shut down. +func TestOpenStateStorePoolRefusesASecondOwner(t *testing.T) { + url := pgtest.URL(t) + first := newBootstrap() + first.cfg = config.Config{DatabaseURL: url} + if err := first.openStateStorePool(t.Context()); err != nil { + t.Fatalf("first openStateStorePool: %v", err) + } + second := newBootstrap() + second.cfg = config.Config{DatabaseURL: url} + err := second.openStateStorePool(t.Context()) + second.cleanup() + first.cleanup() + if !errors.Is(err, postgres.ErrOwned) { + t.Fatalf("second openStateStorePool = %v, want ErrOwned", err) + } + third := newBootstrap() + third.cfg = config.Config{DatabaseURL: url} + if err := third.openStateStorePool(t.Context()); err != nil { + t.Fatalf("openStateStorePool after the owner shut down: %v", err) + } + third.cleanup() +} diff --git a/internal/infrastructure/postgres/ownership.go b/internal/infrastructure/postgres/ownership.go new file mode 100644 index 000000000..fe8d55c83 --- /dev/null +++ b/internal/infrastructure/postgres/ownership.go @@ -0,0 +1,59 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + + "github.com/jackc/pgx/v5/pgxpool" +) + +// ErrOwned is returned when another process already owns the named role on +// this database. +var ErrOwned = errors.New("postgres: owned by another process") + +// Ownership is one process's exclusive claim to serve a role (the State Store, +// the Gateway) against a database. It is a session-level advisory lock held on +// a connection taken out of the pool for the claim's whole life, so a process +// that crashes or loses its connection releases it without cleanup. +type Ownership struct { + conn *pgxpool.Conn + name string + pid uint32 +} + +// AcquireOwnership claims name without blocking. A claim another process holds +// comes back as ErrOwned, never as a wait: a second owner must fail to start, +// not queue behind the first. +func AcquireOwnership(ctx context.Context, pool *pgxpool.Pool, name string) (*Ownership, error) { + conn, err := pool.Acquire(ctx) + if err != nil { + return nil, fmt.Errorf("postgres: ownership %s: %w", name, err) + } + var taken bool + if err := conn.QueryRow(ctx, "SELECT pg_try_advisory_lock(hashtextextended('archie.ownership.' || $1, 0))", name).Scan(&taken); err != nil { + conn.Release() + return nil, fmt.Errorf("postgres: ownership %s: %w", name, err) + } + if !taken { + conn.Release() + return nil, fmt.Errorf("%s: %w", name, ErrOwned) + } + return &Ownership{conn: conn, name: name, pid: conn.Conn().PgConn().PID()}, nil +} + +// Release drops the claim and returns the connection. A connection that +// cannot unlock is closed rather than pooled, which ends the session and with +// it the lock. Release is idempotent. +func (o *Ownership) Release(ctx context.Context) error { + if o == nil || o.conn == nil { + return nil + } + conn := o.conn + o.conn = nil + if _, err := conn.Exec(ctx, "SELECT pg_advisory_unlock(hashtextextended('archie.ownership.' || $1, 0))", o.name); err != nil { + return errors.Join(fmt.Errorf("postgres: release ownership %s: %w", o.name, err), conn.Hijack().Close(ctx)) + } + conn.Release() + return nil +} diff --git a/internal/infrastructure/postgres/ownership_test.go b/internal/infrastructure/postgres/ownership_test.go new file mode 100644 index 000000000..02291fda5 --- /dev/null +++ b/internal/infrastructure/postgres/ownership_test.go @@ -0,0 +1,75 @@ +package postgres + +import ( + "errors" + "testing" + + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/samcharles93/archie-core/internal/infrastructure/postgres/pgtest" +) + +func openPool(t *testing.T, url string) *pgxpool.Pool { + t.Helper() + pool, err := Open(t.Context(), url) + if err != nil { + t.Fatalf("Open: %v", err) + } + t.Cleanup(pool.Close) + return pool +} + +// claimOther takes name as another process would, then optionally releases it +// or kills its backend. +func claimOther(t *testing.T, url, name string, release, terminate bool) { + t.Helper() + other, err := AcquireOwnership(t.Context(), openPool(t, url), name) + if err != nil { + t.Fatalf("first claim: %v", err) + } + t.Cleanup(func() { _ = other.Release(t.Context()) }) + if release { + if err := other.Release(t.Context()); err != nil { + t.Fatalf("Release: %v", err) + } + } + if terminate { + if _, err := openPool(t, url).Exec(t.Context(), "SELECT pg_terminate_backend($1)", other.pid); err != nil { + t.Fatalf("terminate: %v", err) + } + } +} + +func TestAcquireOwnership(t *testing.T) { + tests := []struct { + name string + // held is taken by another process before the claim under test. + held string + // release drops the other claim before the claim under test. + release bool + // terminate kills the other claim's backend, as a crash would. + terminate bool + want error + }{ + {name: "state-store", want: nil}, + {name: "state-store", held: "state-store", want: ErrOwned}, + {name: "state-store", held: "state-store", release: true, want: nil}, + {name: "state-store", held: "state-store", terminate: true, want: nil}, + {name: "gateway", held: "state-store", want: nil}, + } + for _, tt := range tests { + t.Run(tt.name+"/held="+tt.held, func(t *testing.T) { + url := pgtest.URL(t) + if tt.held != "" { + claimOther(t, url, tt.held, tt.release, tt.terminate) + } + got, err := AcquireOwnership(t.Context(), openPool(t, url), tt.name) + if got != nil { + t.Cleanup(func() { _ = got.Release(t.Context()) }) + } + if !errors.Is(err, tt.want) { + t.Fatalf("AcquireOwnership(%q) = %v, want %v", tt.name, err, tt.want) + } + }) + } +}