fix(pg): hold one advisory lock across migration and grants (#2082)
This commit is contained in:
3 files changed
+55
-18
No files matched your search
@@ -26,6 +26,7 @@ const (
|
||||
const (
|
||||
stageSetupRead = "cannot read whether owner setup is required"
|
||||
stageStartup = "starting operations failed"
|
||||
stageMigrate = "migrating the database failed"
|
||||
)
|
||||
|
||||
// How long a failed boot step waits before it runs again. The delay doubles up
|
||||
@@ -172,7 +173,7 @@ func activate(ctx context.Context, log *slog.Logger, lookup setupLookup, done <-
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
if err := retryStartup(ctx, log, g, stageStartup, ops.Start); err != nil {
|
||||
if err := retryStartup(ctx, log, g.stall, stageStartup, ops.Start); err != nil {
|
||||
return err
|
||||
}
|
||||
g.publish()
|
||||
@@ -207,20 +208,20 @@ func waitForSetup(ctx context.Context, log *slog.Logger, lookup setupLookup, don
|
||||
}
|
||||
}
|
||||
|
||||
// retryStartup runs a boot step until it succeeds, recording the stage that
|
||||
// failed for readiness. A boot step that reads the database must not turn a brief
|
||||
// outage into a terminated process.
|
||||
func retryStartup(ctx context.Context, log *slog.Logger, g *gate, stage string, attempt func(context.Context) error) error {
|
||||
// retryStartup runs a boot step until it succeeds, passing the stage that
|
||||
// failed to mark so readiness can report it. A boot step that reads the database
|
||||
// must not turn a brief outage into a terminated process.
|
||||
func retryStartup(ctx context.Context, log *slog.Logger, mark func(stage string), stage string, attempt func(context.Context) error) error {
|
||||
for delay := startupRetryMin; ; delay = min(delay*2, startupRetryMax) {
|
||||
err := attempt(ctx)
|
||||
if err == nil {
|
||||
g.stall("")
|
||||
mark("")
|
||||
return nil
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
g.stall(stage)
|
||||
mark(stage)
|
||||
log.WarnContext(ctx, "startup step failed; retrying", "stage", stage, "err", err, "in", delay)
|
||||
select {
|
||||
case <-time.After(delay):
|
||||
|
||||
@@ -76,7 +76,11 @@ func Run(ctx context.Context, configPath string) error {
|
||||
return fmt.Errorf("claim the database: %w", err)
|
||||
}
|
||||
defer holdServe()
|
||||
if err := pg.Migrate(ctx, log, cfg.Database.OwnerURL, cfg.Database.URL, schemas); err != nil {
|
||||
// Nothing is served yet, so there is no readiness to report while this retries.
|
||||
migrate := func(ctx context.Context) error {
|
||||
return pg.Migrate(ctx, log, cfg.Database.OwnerURL, cfg.Database.URL, schemas)
|
||||
}
|
||||
if err := retryStartup(ctx, log, func(string) {}, stageMigrate, migrate); err != nil {
|
||||
_ = db.Close()
|
||||
return err
|
||||
}
|
||||
|
||||
+42
-10
@@ -15,7 +15,6 @@ import (
|
||||
_ "github.com/jackc/pgx/v5/stdlib"
|
||||
"github.com/jmoiron/sqlx"
|
||||
"github.com/pressly/goose/v3"
|
||||
"github.com/pressly/goose/v3/lock"
|
||||
)
|
||||
|
||||
//go:embed migrations
|
||||
@@ -34,10 +33,17 @@ func Open(ctx context.Context, url string) (*sqlx.DB, error) {
|
||||
return db, nil
|
||||
}
|
||||
|
||||
// Migrate applies every pending migration as the owner, under a session
|
||||
// advisory lock, so two instances starting together never race, then grants
|
||||
// the runtime role DML on each module schema. A failed migration returns an
|
||||
// error and the boot stops.
|
||||
// migrateLockName names the advisory lock that serialises migration and
|
||||
// privilege grants across instances.
|
||||
const migrateLockName = "migrate"
|
||||
|
||||
// Migrate applies every pending migration as the owner, then grants the
|
||||
// runtime role DML on each module schema. Both steps run under one session
|
||||
// advisory lock, so two instances starting together take turns: the second
|
||||
// finds the migrations applied and the grants already in place. Granting on the
|
||||
// same object from two sessions fails with "tuple concurrently updated", so the
|
||||
// grants need the lock as much as the migrations do. It returns an error on any
|
||||
// failure, and the caller retries.
|
||||
func Migrate(ctx context.Context, log *slog.Logger, ownerURL, runtimeURL string, schemas []string) error {
|
||||
db, err := Open(ctx, ownerURL)
|
||||
if err != nil {
|
||||
@@ -45,15 +51,17 @@ func Migrate(ctx context.Context, log *slog.Logger, ownerURL, runtimeURL string,
|
||||
}
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
release, err := lockMigrate(ctx, db)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer release()
|
||||
|
||||
files, err := fs.Sub(migrationFiles, "migrations")
|
||||
if err != nil {
|
||||
return fmt.Errorf("migration files: %w", err)
|
||||
}
|
||||
locker, err := lock.NewPostgresSessionLocker()
|
||||
if err != nil {
|
||||
return fmt.Errorf("migration lock: %w", err)
|
||||
}
|
||||
provider, err := goose.NewProvider(goose.DialectPostgres, db.DB, files, goose.WithSessionLocker(locker))
|
||||
provider, err := goose.NewProvider(goose.DialectPostgres, db.DB, files)
|
||||
if err != nil {
|
||||
return fmt.Errorf("migration provider: %w", err)
|
||||
}
|
||||
@@ -67,6 +75,30 @@ func Migrate(ctx context.Context, log *slog.Logger, ownerURL, runtimeURL string,
|
||||
return grantRuntime(ctx, db, runtimeURL, schemas)
|
||||
}
|
||||
|
||||
// lockMigrate waits for the migration advisory lock and holds it on a dedicated
|
||||
// connection until release is called. Goose's own session locker covers only
|
||||
// the migrations, which leaves the grants unprotected, so the lock is taken here
|
||||
// around both.
|
||||
func lockMigrate(ctx context.Context, db *sqlx.DB) (release func(), err error) {
|
||||
conn, err := db.Conn(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("lock migrations: %w", err)
|
||||
}
|
||||
if _, err := conn.ExecContext(ctx, "SELECT pg_advisory_lock(hashtextextended($1, 0))", migrateLockKey()); err != nil {
|
||||
_ = conn.Close()
|
||||
return nil, fmt.Errorf("lock migrations: %w", err)
|
||||
}
|
||||
return func() {
|
||||
// Closing the connection would release the lock too, but the pool may
|
||||
// keep it open.
|
||||
_, _ = conn.ExecContext(context.WithoutCancel(ctx),
|
||||
"SELECT pg_advisory_unlock(hashtextextended($1, 0))", migrateLockKey())
|
||||
_ = conn.Close()
|
||||
}, nil
|
||||
}
|
||||
|
||||
func migrateLockKey() string { return "archie." + migrateLockName + ".owner" }
|
||||
|
||||
// grantRuntime gives the runtime role USAGE and DML on each schema, never DDL.
|
||||
func grantRuntime(ctx context.Context, db *sqlx.DB, runtimeURL string, schemas []string) error {
|
||||
cfg, err := pgx.ParseConfig(runtimeURL)
|
||||
|
||||
Reference in new issue
Block a user