fix(pg): hold one advisory lock across migration and grants (#2082)

This commit is contained in:
sam committed 2026-11-21 14:53:17 +11:00
1 parent 7b6154e7e8
commit 2e48acfc92
3 files changed
+55 -18

No files matched your search

+8 -7
View File
@@ -26,6 +26,7 @@ const (
const ( const (
stageSetupRead = "cannot read whether owner setup is required" stageSetupRead = "cannot read whether owner setup is required"
stageStartup = "starting operations failed" stageStartup = "starting operations failed"
stageMigrate = "migrating the database failed"
) )
// How long a failed boot step waits before it runs again. The delay doubles up // How long a failed boot step waits before it runs again. The delay doubles up
@@ -172,7 +173,7 @@ func activate(ctx context.Context, log *slog.Logger, lookup setupLookup, done <-
return ctx.Err() return ctx.Err()
} }
} }
if err := retryStartup(ctx, log, g, stageStartup, ops.Start); err != nil { if err := retryStartup(ctx, log, g.stall, stageStartup, ops.Start); err != nil {
return err return err
} }
g.publish() g.publish()
@@ -207,20 +208,20 @@ func waitForSetup(ctx context.Context, log *slog.Logger, lookup setupLookup, don
} }
} }
// retryStartup runs a boot step until it succeeds, recording the stage that // retryStartup runs a boot step until it succeeds, passing the stage that
// failed for readiness. A boot step that reads the database must not turn a brief // failed to mark so readiness can report it. A boot step that reads the database
// outage into a terminated process. // must not turn a brief outage into a terminated process.
func retryStartup(ctx context.Context, log *slog.Logger, g *gate, stage string, attempt func(context.Context) error) error { func retryStartup(ctx context.Context, log *slog.Logger, mark func(stage string), stage string, attempt func(context.Context) error) error {
for delay := startupRetryMin; ; delay = min(delay*2, startupRetryMax) { for delay := startupRetryMin; ; delay = min(delay*2, startupRetryMax) {
err := attempt(ctx) err := attempt(ctx)
if err == nil { if err == nil {
g.stall("") mark("")
return nil return nil
} }
if ctx.Err() != nil { if ctx.Err() != nil {
return ctx.Err() return ctx.Err()
} }
g.stall(stage) mark(stage)
log.WarnContext(ctx, "startup step failed; retrying", "stage", stage, "err", err, "in", delay) log.WarnContext(ctx, "startup step failed; retrying", "stage", stage, "err", err, "in", delay)
select { select {
case <-time.After(delay): case <-time.After(delay):
+5 -1
View File
@@ -76,7 +76,11 @@ func Run(ctx context.Context, configPath string) error {
return fmt.Errorf("claim the database: %w", err) return fmt.Errorf("claim the database: %w", err)
} }
defer holdServe() defer holdServe()
if err := pg.Migrate(ctx, log, cfg.Database.OwnerURL, cfg.Database.URL, schemas); err != nil { // Nothing is served yet, so there is no readiness to report while this retries.
migrate := func(ctx context.Context) error {
return pg.Migrate(ctx, log, cfg.Database.OwnerURL, cfg.Database.URL, schemas)
}
if err := retryStartup(ctx, log, func(string) {}, stageMigrate, migrate); err != nil {
_ = db.Close() _ = db.Close()
return err return err
} }
+42 -10
View File
@@ -15,7 +15,6 @@ import (
_ "github.com/jackc/pgx/v5/stdlib" _ "github.com/jackc/pgx/v5/stdlib"
"github.com/jmoiron/sqlx" "github.com/jmoiron/sqlx"
"github.com/pressly/goose/v3" "github.com/pressly/goose/v3"
"github.com/pressly/goose/v3/lock"
) )
//go:embed migrations //go:embed migrations
@@ -34,10 +33,17 @@ func Open(ctx context.Context, url string) (*sqlx.DB, error) {
return db, nil return db, nil
} }
// Migrate applies every pending migration as the owner, under a session // migrateLockName names the advisory lock that serialises migration and
// advisory lock, so two instances starting together never race, then grants // privilege grants across instances.
// the runtime role DML on each module schema. A failed migration returns an const migrateLockName = "migrate"
// error and the boot stops.
// Migrate applies every pending migration as the owner, then grants the
// runtime role DML on each module schema. Both steps run under one session
// advisory lock, so two instances starting together take turns: the second
// finds the migrations applied and the grants already in place. Granting on the
// same object from two sessions fails with "tuple concurrently updated", so the
// grants need the lock as much as the migrations do. It returns an error on any
// failure, and the caller retries.
func Migrate(ctx context.Context, log *slog.Logger, ownerURL, runtimeURL string, schemas []string) error { func Migrate(ctx context.Context, log *slog.Logger, ownerURL, runtimeURL string, schemas []string) error {
db, err := Open(ctx, ownerURL) db, err := Open(ctx, ownerURL)
if err != nil { if err != nil {
@@ -45,15 +51,17 @@ func Migrate(ctx context.Context, log *slog.Logger, ownerURL, runtimeURL string,
} }
defer func() { _ = db.Close() }() defer func() { _ = db.Close() }()
release, err := lockMigrate(ctx, db)
if err != nil {
return err
}
defer release()
files, err := fs.Sub(migrationFiles, "migrations") files, err := fs.Sub(migrationFiles, "migrations")
if err != nil { if err != nil {
return fmt.Errorf("migration files: %w", err) return fmt.Errorf("migration files: %w", err)
} }
locker, err := lock.NewPostgresSessionLocker() provider, err := goose.NewProvider(goose.DialectPostgres, db.DB, files)
if err != nil {
return fmt.Errorf("migration lock: %w", err)
}
provider, err := goose.NewProvider(goose.DialectPostgres, db.DB, files, goose.WithSessionLocker(locker))
if err != nil { if err != nil {
return fmt.Errorf("migration provider: %w", err) return fmt.Errorf("migration provider: %w", err)
} }
@@ -67,6 +75,30 @@ func Migrate(ctx context.Context, log *slog.Logger, ownerURL, runtimeURL string,
return grantRuntime(ctx, db, runtimeURL, schemas) return grantRuntime(ctx, db, runtimeURL, schemas)
} }
// lockMigrate waits for the migration advisory lock and holds it on a dedicated
// connection until release is called. Goose's own session locker covers only
// the migrations, which leaves the grants unprotected, so the lock is taken here
// around both.
func lockMigrate(ctx context.Context, db *sqlx.DB) (release func(), err error) {
conn, err := db.Conn(ctx)
if err != nil {
return nil, fmt.Errorf("lock migrations: %w", err)
}
if _, err := conn.ExecContext(ctx, "SELECT pg_advisory_lock(hashtextextended($1, 0))", migrateLockKey()); err != nil {
_ = conn.Close()
return nil, fmt.Errorf("lock migrations: %w", err)
}
return func() {
// Closing the connection would release the lock too, but the pool may
// keep it open.
_, _ = conn.ExecContext(context.WithoutCancel(ctx),
"SELECT pg_advisory_unlock(hashtextextended($1, 0))", migrateLockKey())
_ = conn.Close()
}, nil
}
func migrateLockKey() string { return "archie." + migrateLockName + ".owner" }
// grantRuntime gives the runtime role USAGE and DML on each schema, never DDL. // grantRuntime gives the runtime role USAGE and DML on each schema, never DDL.
func grantRuntime(ctx context.Context, db *sqlx.DB, runtimeURL string, schemas []string) error { func grantRuntime(ctx context.Context, db *sqlx.DB, runtimeURL string, schemas []string) error {
cfg, err := pgx.ParseConfig(runtimeURL) cfg, err := pgx.ParseConfig(runtimeURL)