feat(postgres): State Store claims serve ownership with a session advisory lock
This commit is contained in:
1 parent
972355fc63
commit
a71d062b4a
4 files changed
+176
No files matched your search
@@ -13,6 +13,9 @@ import (
|
||||
// from starting rather than falling back to SQLite, and a migration failure is
|
||||
// a startup failure, not a warning.
|
||||
//
|
||||
// It then claims serve ownership of the database for the process's life, so a
|
||||
// second State Store against the same database refuses to start.
|
||||
//
|
||||
// The pool is stored on boot.pg and closed at shutdown; nothing opens a second
|
||||
// pool per subsystem.
|
||||
func (b *boot) openStateStorePool(ctx context.Context) error {
|
||||
@@ -32,7 +35,17 @@ func (b *boot) openStateStorePool(ctx context.Context) error {
|
||||
b.log.Error("migrate state store postgres schema", "err", err)
|
||||
return err
|
||||
}
|
||||
ownership, err := postgres.AcquireOwnership(ctx, pool, "state-store")
|
||||
if err != nil {
|
||||
pool.Close()
|
||||
b.log.Error("claim state store ownership", "err", err)
|
||||
return err
|
||||
}
|
||||
b.pg = pool
|
||||
b.addCleanup(pool.Close)
|
||||
// Registered after the pool so it runs first: the claim is dropped before
|
||||
// the pool closes.
|
||||
releaseCtx := context.WithoutCancel(ctx)
|
||||
b.addCleanup(func() { _ = ownership.Release(releaseCtx) })
|
||||
return nil
|
||||
}
|
||||
@@ -1,10 +1,13 @@
|
||||
package archied
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/postgres"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/postgres/pgtest"
|
||||
)
|
||||
|
||||
// TestOpenStateStorePoolFailsClosedWithoutURL pins D1.2: the State Store is
|
||||
@@ -30,3 +33,29 @@ func TestOpenStateStorePoolFailsClosedOnInvalidURL(t *testing.T) {
|
||||
t.Fatal("openStateStorePool with an unparseable database_url succeeded; want fail closed")
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenStateStorePoolRefusesASecondOwner pins the serve-ownership claim: a
|
||||
// second State Store against the same database fails to start while the first
|
||||
// holds it, and starts once the first has shut down.
|
||||
func TestOpenStateStorePoolRefusesASecondOwner(t *testing.T) {
|
||||
url := pgtest.URL(t)
|
||||
first := newBootstrap()
|
||||
first.cfg = config.Config{DatabaseURL: url}
|
||||
if err := first.openStateStorePool(t.Context()); err != nil {
|
||||
t.Fatalf("first openStateStorePool: %v", err)
|
||||
}
|
||||
second := newBootstrap()
|
||||
second.cfg = config.Config{DatabaseURL: url}
|
||||
err := second.openStateStorePool(t.Context())
|
||||
second.cleanup()
|
||||
first.cleanup()
|
||||
if !errors.Is(err, postgres.ErrOwned) {
|
||||
t.Fatalf("second openStateStorePool = %v, want ErrOwned", err)
|
||||
}
|
||||
third := newBootstrap()
|
||||
third.cfg = config.Config{DatabaseURL: url}
|
||||
if err := third.openStateStorePool(t.Context()); err != nil {
|
||||
t.Fatalf("openStateStorePool after the owner shut down: %v", err)
|
||||
}
|
||||
third.cleanup()
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// ErrOwned is returned when another process already owns the named role on
|
||||
// this database.
|
||||
var ErrOwned = errors.New("postgres: owned by another process")
|
||||
|
||||
// Ownership is one process's exclusive claim to serve a role (the State Store,
|
||||
// the Gateway) against a database. It is a session-level advisory lock held on
|
||||
// a connection taken out of the pool for the claim's whole life, so a process
|
||||
// that crashes or loses its connection releases it without cleanup.
|
||||
type Ownership struct {
|
||||
conn *pgxpool.Conn
|
||||
name string
|
||||
pid uint32
|
||||
}
|
||||
|
||||
// AcquireOwnership claims name without blocking. A claim another process holds
|
||||
// comes back as ErrOwned, never as a wait: a second owner must fail to start,
|
||||
// not queue behind the first.
|
||||
func AcquireOwnership(ctx context.Context, pool *pgxpool.Pool, name string) (*Ownership, error) {
|
||||
conn, err := pool.Acquire(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("postgres: ownership %s: %w", name, err)
|
||||
}
|
||||
var taken bool
|
||||
if err := conn.QueryRow(ctx, "SELECT pg_try_advisory_lock(hashtextextended('archie.ownership.' || $1, 0))", name).Scan(&taken); err != nil {
|
||||
conn.Release()
|
||||
return nil, fmt.Errorf("postgres: ownership %s: %w", name, err)
|
||||
}
|
||||
if !taken {
|
||||
conn.Release()
|
||||
return nil, fmt.Errorf("%s: %w", name, ErrOwned)
|
||||
}
|
||||
return &Ownership{conn: conn, name: name, pid: conn.Conn().PgConn().PID()}, nil
|
||||
}
|
||||
|
||||
// Release drops the claim and returns the connection. A connection that
|
||||
// cannot unlock is closed rather than pooled, which ends the session and with
|
||||
// it the lock. Release is idempotent.
|
||||
func (o *Ownership) Release(ctx context.Context) error {
|
||||
if o == nil || o.conn == nil {
|
||||
return nil
|
||||
}
|
||||
conn := o.conn
|
||||
o.conn = nil
|
||||
if _, err := conn.Exec(ctx, "SELECT pg_advisory_unlock(hashtextextended('archie.ownership.' || $1, 0))", o.name); err != nil {
|
||||
return errors.Join(fmt.Errorf("postgres: release ownership %s: %w", o.name, err), conn.Hijack().Close(ctx))
|
||||
}
|
||||
conn.Release()
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/postgres/pgtest"
|
||||
)
|
||||
|
||||
func openPool(t *testing.T, url string) *pgxpool.Pool {
|
||||
t.Helper()
|
||||
pool, err := Open(t.Context(), url)
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
t.Cleanup(pool.Close)
|
||||
return pool
|
||||
}
|
||||
|
||||
// claimOther takes name as another process would, then optionally releases it
|
||||
// or kills its backend.
|
||||
func claimOther(t *testing.T, url, name string, release, terminate bool) {
|
||||
t.Helper()
|
||||
other, err := AcquireOwnership(t.Context(), openPool(t, url), name)
|
||||
if err != nil {
|
||||
t.Fatalf("first claim: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = other.Release(t.Context()) })
|
||||
if release {
|
||||
if err := other.Release(t.Context()); err != nil {
|
||||
t.Fatalf("Release: %v", err)
|
||||
}
|
||||
}
|
||||
if terminate {
|
||||
if _, err := openPool(t, url).Exec(t.Context(), "SELECT pg_terminate_backend($1)", other.pid); err != nil {
|
||||
t.Fatalf("terminate: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcquireOwnership(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
// held is taken by another process before the claim under test.
|
||||
held string
|
||||
// release drops the other claim before the claim under test.
|
||||
release bool
|
||||
// terminate kills the other claim's backend, as a crash would.
|
||||
terminate bool
|
||||
want error
|
||||
}{
|
||||
{name: "state-store", want: nil},
|
||||
{name: "state-store", held: "state-store", want: ErrOwned},
|
||||
{name: "state-store", held: "state-store", release: true, want: nil},
|
||||
{name: "state-store", held: "state-store", terminate: true, want: nil},
|
||||
{name: "gateway", held: "state-store", want: nil},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name+"/held="+tt.held, func(t *testing.T) {
|
||||
url := pgtest.URL(t)
|
||||
if tt.held != "" {
|
||||
claimOther(t, url, tt.held, tt.release, tt.terminate)
|
||||
}
|
||||
got, err := AcquireOwnership(t.Context(), openPool(t, url), tt.name)
|
||||
if got != nil {
|
||||
t.Cleanup(func() { _ = got.Release(t.Context()) })
|
||||
}
|
||||
if !errors.Is(err, tt.want) {
|
||||
t.Fatalf("AcquireOwnership(%q) = %v, want %v", tt.name, err, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user