diff --git a/internal/app/archie/gate.go b/internal/app/archie/gate.go index 1bab3f746..955aec5ae 100644 --- a/internal/app/archie/gate.go +++ b/internal/app/archie/gate.go @@ -26,6 +26,7 @@ const ( const ( stageSetupRead = "cannot read whether owner setup is required" stageStartup = "starting operations failed" + stageMigrate = "migrating the database failed" ) // How long a failed boot step waits before it runs again. The delay doubles up @@ -172,7 +173,7 @@ func activate(ctx context.Context, log *slog.Logger, lookup setupLookup, done <- return ctx.Err() } } - if err := retryStartup(ctx, log, g, stageStartup, ops.Start); err != nil { + if err := retryStartup(ctx, log, g.stall, stageStartup, ops.Start); err != nil { return err } g.publish() @@ -207,20 +208,20 @@ func waitForSetup(ctx context.Context, log *slog.Logger, lookup setupLookup, don } } -// retryStartup runs a boot step until it succeeds, recording the stage that -// failed for readiness. A boot step that reads the database must not turn a brief -// outage into a terminated process. -func retryStartup(ctx context.Context, log *slog.Logger, g *gate, stage string, attempt func(context.Context) error) error { +// retryStartup runs a boot step until it succeeds, passing the stage that +// failed to mark so readiness can report it. A boot step that reads the database +// must not turn a brief outage into a terminated process. +func retryStartup(ctx context.Context, log *slog.Logger, mark func(stage string), stage string, attempt func(context.Context) error) error { for delay := startupRetryMin; ; delay = min(delay*2, startupRetryMax) { err := attempt(ctx) if err == nil { - g.stall("") + mark("") return nil } if ctx.Err() != nil { return ctx.Err() } - g.stall(stage) + mark(stage) log.WarnContext(ctx, "startup step failed; retrying", "stage", stage, "err", err, "in", delay) select { case <-time.After(delay): diff --git a/internal/app/archie/run.go b/internal/app/archie/run.go index 13b566942..eef739f6f 100644 --- a/internal/app/archie/run.go +++ b/internal/app/archie/run.go @@ -76,7 +76,11 @@ func Run(ctx context.Context, configPath string) error { return fmt.Errorf("claim the database: %w", err) } defer holdServe() - if err := pg.Migrate(ctx, log, cfg.Database.OwnerURL, cfg.Database.URL, schemas); err != nil { + // Nothing is served yet, so there is no readiness to report while this retries. + migrate := func(ctx context.Context) error { + return pg.Migrate(ctx, log, cfg.Database.OwnerURL, cfg.Database.URL, schemas) + } + if err := retryStartup(ctx, log, func(string) {}, stageMigrate, migrate); err != nil { _ = db.Close() return err } diff --git a/internal/platform/pg/pg.go b/internal/platform/pg/pg.go index e5d58da90..a37f7d6ce 100644 --- a/internal/platform/pg/pg.go +++ b/internal/platform/pg/pg.go @@ -15,7 +15,6 @@ import ( _ "github.com/jackc/pgx/v5/stdlib" "github.com/jmoiron/sqlx" "github.com/pressly/goose/v3" - "github.com/pressly/goose/v3/lock" ) //go:embed migrations @@ -34,10 +33,17 @@ func Open(ctx context.Context, url string) (*sqlx.DB, error) { return db, nil } -// Migrate applies every pending migration as the owner, under a session -// advisory lock, so two instances starting together never race, then grants -// the runtime role DML on each module schema. A failed migration returns an -// error and the boot stops. +// migrateLockName names the advisory lock that serialises migration and +// privilege grants across instances. +const migrateLockName = "migrate" + +// Migrate applies every pending migration as the owner, then grants the +// runtime role DML on each module schema. Both steps run under one session +// advisory lock, so two instances starting together take turns: the second +// finds the migrations applied and the grants already in place. Granting on the +// same object from two sessions fails with "tuple concurrently updated", so the +// grants need the lock as much as the migrations do. It returns an error on any +// failure, and the caller retries. func Migrate(ctx context.Context, log *slog.Logger, ownerURL, runtimeURL string, schemas []string) error { db, err := Open(ctx, ownerURL) if err != nil { @@ -45,15 +51,17 @@ func Migrate(ctx context.Context, log *slog.Logger, ownerURL, runtimeURL string, } defer func() { _ = db.Close() }() + release, err := lockMigrate(ctx, db) + if err != nil { + return err + } + defer release() + files, err := fs.Sub(migrationFiles, "migrations") if err != nil { return fmt.Errorf("migration files: %w", err) } - locker, err := lock.NewPostgresSessionLocker() - if err != nil { - return fmt.Errorf("migration lock: %w", err) - } - provider, err := goose.NewProvider(goose.DialectPostgres, db.DB, files, goose.WithSessionLocker(locker)) + provider, err := goose.NewProvider(goose.DialectPostgres, db.DB, files) if err != nil { return fmt.Errorf("migration provider: %w", err) } @@ -67,6 +75,30 @@ func Migrate(ctx context.Context, log *slog.Logger, ownerURL, runtimeURL string, return grantRuntime(ctx, db, runtimeURL, schemas) } +// lockMigrate waits for the migration advisory lock and holds it on a dedicated +// connection until release is called. Goose's own session locker covers only +// the migrations, which leaves the grants unprotected, so the lock is taken here +// around both. +func lockMigrate(ctx context.Context, db *sqlx.DB) (release func(), err error) { + conn, err := db.Conn(ctx) + if err != nil { + return nil, fmt.Errorf("lock migrations: %w", err) + } + if _, err := conn.ExecContext(ctx, "SELECT pg_advisory_lock(hashtextextended($1, 0))", migrateLockKey()); err != nil { + _ = conn.Close() + return nil, fmt.Errorf("lock migrations: %w", err) + } + return func() { + // Closing the connection would release the lock too, but the pool may + // keep it open. + _, _ = conn.ExecContext(context.WithoutCancel(ctx), + "SELECT pg_advisory_unlock(hashtextextended($1, 0))", migrateLockKey()) + _ = conn.Close() + }, nil +} + +func migrateLockKey() string { return "archie." + migrateLockName + ".owner" } + // grantRuntime gives the runtime role USAGE and DML on each schema, never DDL. func grantRuntime(ctx context.Context, db *sqlx.DB, runtimeURL string, schemas []string) error { cfg, err := pgx.ParseConfig(runtimeURL)