fix(pg): hold one advisory lock across migration and grants (#2082)

This commit is contained in:
sam committed 2026-11-21 14:53:17 +11:00
1 parent 7b6154e7e8
commit 2e48acfc92
3 files changed
+55 -18

No files matched your search

+8 -7
View File
@@ -26,6 +26,7 @@ const (
const (
stageSetupRead = "cannot read whether owner setup is required"
stageStartup = "starting operations failed"
stageMigrate = "migrating the database failed"
)
// How long a failed boot step waits before it runs again. The delay doubles up
@@ -172,7 +173,7 @@ func activate(ctx context.Context, log *slog.Logger, lookup setupLookup, done <-
return ctx.Err()
}
}
if err := retryStartup(ctx, log, g, stageStartup, ops.Start); err != nil {
if err := retryStartup(ctx, log, g.stall, stageStartup, ops.Start); err != nil {
return err
}
g.publish()
@@ -207,20 +208,20 @@ func waitForSetup(ctx context.Context, log *slog.Logger, lookup setupLookup, don
}
}
// retryStartup runs a boot step until it succeeds, recording the stage that
// failed for readiness. A boot step that reads the database must not turn a brief
// outage into a terminated process.
func retryStartup(ctx context.Context, log *slog.Logger, g *gate, stage string, attempt func(context.Context) error) error {
// retryStartup runs a boot step until it succeeds, passing the stage that
// failed to mark so readiness can report it. A boot step that reads the database
// must not turn a brief outage into a terminated process.
func retryStartup(ctx context.Context, log *slog.Logger, mark func(stage string), stage string, attempt func(context.Context) error) error {
for delay := startupRetryMin; ; delay = min(delay*2, startupRetryMax) {
err := attempt(ctx)
if err == nil {
g.stall("")
mark("")
return nil
}
if ctx.Err() != nil {
return ctx.Err()
}
g.stall(stage)
mark(stage)
log.WarnContext(ctx, "startup step failed; retrying", "stage", stage, "err", err, "in", delay)
select {
case <-time.After(delay):
+5 -1
View File
@@ -76,7 +76,11 @@ func Run(ctx context.Context, configPath string) error {
return fmt.Errorf("claim the database: %w", err)
}
defer holdServe()
if err := pg.Migrate(ctx, log, cfg.Database.OwnerURL, cfg.Database.URL, schemas); err != nil {
// Nothing is served yet, so there is no readiness to report while this retries.
migrate := func(ctx context.Context) error {
return pg.Migrate(ctx, log, cfg.Database.OwnerURL, cfg.Database.URL, schemas)
}
if err := retryStartup(ctx, log, func(string) {}, stageMigrate, migrate); err != nil {
_ = db.Close()
return err
}
+42 -10
View File
@@ -15,7 +15,6 @@ import (
_ "github.com/jackc/pgx/v5/stdlib"
"github.com/jmoiron/sqlx"
"github.com/pressly/goose/v3"
"github.com/pressly/goose/v3/lock"
)
//go:embed migrations
@@ -34,10 +33,17 @@ func Open(ctx context.Context, url string) (*sqlx.DB, error) {
return db, nil
}
// Migrate applies every pending migration as the owner, under a session
// advisory lock, so two instances starting together never race, then grants
// the runtime role DML on each module schema. A failed migration returns an
// error and the boot stops.
// migrateLockName names the advisory lock that serialises migration and
// privilege grants across instances.
const migrateLockName = "migrate"
// Migrate applies every pending migration as the owner, then grants the
// runtime role DML on each module schema. Both steps run under one session
// advisory lock, so two instances starting together take turns: the second
// finds the migrations applied and the grants already in place. Granting on the
// same object from two sessions fails with "tuple concurrently updated", so the
// grants need the lock as much as the migrations do. It returns an error on any
// failure, and the caller retries.
func Migrate(ctx context.Context, log *slog.Logger, ownerURL, runtimeURL string, schemas []string) error {
db, err := Open(ctx, ownerURL)
if err != nil {
@@ -45,15 +51,17 @@ func Migrate(ctx context.Context, log *slog.Logger, ownerURL, runtimeURL string,
}
defer func() { _ = db.Close() }()
release, err := lockMigrate(ctx, db)
if err != nil {
return err
}
defer release()
files, err := fs.Sub(migrationFiles, "migrations")
if err != nil {
return fmt.Errorf("migration files: %w", err)
}
locker, err := lock.NewPostgresSessionLocker()
if err != nil {
return fmt.Errorf("migration lock: %w", err)
}
provider, err := goose.NewProvider(goose.DialectPostgres, db.DB, files, goose.WithSessionLocker(locker))
provider, err := goose.NewProvider(goose.DialectPostgres, db.DB, files)
if err != nil {
return fmt.Errorf("migration provider: %w", err)
}
@@ -67,6 +75,30 @@ func Migrate(ctx context.Context, log *slog.Logger, ownerURL, runtimeURL string,
return grantRuntime(ctx, db, runtimeURL, schemas)
}
// lockMigrate waits for the migration advisory lock and holds it on a dedicated
// connection until release is called. Goose's own session locker covers only
// the migrations, which leaves the grants unprotected, so the lock is taken here
// around both.
func lockMigrate(ctx context.Context, db *sqlx.DB) (release func(), err error) {
conn, err := db.Conn(ctx)
if err != nil {
return nil, fmt.Errorf("lock migrations: %w", err)
}
if _, err := conn.ExecContext(ctx, "SELECT pg_advisory_lock(hashtextextended($1, 0))", migrateLockKey()); err != nil {
_ = conn.Close()
return nil, fmt.Errorf("lock migrations: %w", err)
}
return func() {
// Closing the connection would release the lock too, but the pool may
// keep it open.
_, _ = conn.ExecContext(context.WithoutCancel(ctx),
"SELECT pg_advisory_unlock(hashtextextended($1, 0))", migrateLockKey())
_ = conn.Close()
}, nil
}
func migrateLockKey() string { return "archie." + migrateLockName + ".owner" }
// grantRuntime gives the runtime role USAGE and DML on each schema, never DDL.
func grantRuntime(ctx context.Context, db *sqlx.DB, runtimeURL string, schemas []string) error {
cfg, err := pgx.ParseConfig(runtimeURL)