fix(pg): hold one advisory lock across migration and grants (#2082)
This commit is contained in:
3 files changed
+55
-18
No files matched your search
@@ -26,6 +26,7 @@ const (
|
|||||||
const (
|
const (
|
||||||
stageSetupRead = "cannot read whether owner setup is required"
|
stageSetupRead = "cannot read whether owner setup is required"
|
||||||
stageStartup = "starting operations failed"
|
stageStartup = "starting operations failed"
|
||||||
|
stageMigrate = "migrating the database failed"
|
||||||
)
|
)
|
||||||
|
|
||||||
// How long a failed boot step waits before it runs again. The delay doubles up
|
// How long a failed boot step waits before it runs again. The delay doubles up
|
||||||
@@ -172,7 +173,7 @@ func activate(ctx context.Context, log *slog.Logger, lookup setupLookup, done <-
|
|||||||
return ctx.Err()
|
return ctx.Err()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if err := retryStartup(ctx, log, g, stageStartup, ops.Start); err != nil {
|
if err := retryStartup(ctx, log, g.stall, stageStartup, ops.Start); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
g.publish()
|
g.publish()
|
||||||
@@ -207,20 +208,20 @@ func waitForSetup(ctx context.Context, log *slog.Logger, lookup setupLookup, don
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// retryStartup runs a boot step until it succeeds, recording the stage that
|
// retryStartup runs a boot step until it succeeds, passing the stage that
|
||||||
// failed for readiness. A boot step that reads the database must not turn a brief
|
// failed to mark so readiness can report it. A boot step that reads the database
|
||||||
// outage into a terminated process.
|
// must not turn a brief outage into a terminated process.
|
||||||
func retryStartup(ctx context.Context, log *slog.Logger, g *gate, stage string, attempt func(context.Context) error) error {
|
func retryStartup(ctx context.Context, log *slog.Logger, mark func(stage string), stage string, attempt func(context.Context) error) error {
|
||||||
for delay := startupRetryMin; ; delay = min(delay*2, startupRetryMax) {
|
for delay := startupRetryMin; ; delay = min(delay*2, startupRetryMax) {
|
||||||
err := attempt(ctx)
|
err := attempt(ctx)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
g.stall("")
|
mark("")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
if ctx.Err() != nil {
|
if ctx.Err() != nil {
|
||||||
return ctx.Err()
|
return ctx.Err()
|
||||||
}
|
}
|
||||||
g.stall(stage)
|
mark(stage)
|
||||||
log.WarnContext(ctx, "startup step failed; retrying", "stage", stage, "err", err, "in", delay)
|
log.WarnContext(ctx, "startup step failed; retrying", "stage", stage, "err", err, "in", delay)
|
||||||
select {
|
select {
|
||||||
case <-time.After(delay):
|
case <-time.After(delay):
|
||||||
|
|||||||
@@ -76,7 +76,11 @@ func Run(ctx context.Context, configPath string) error {
|
|||||||
return fmt.Errorf("claim the database: %w", err)
|
return fmt.Errorf("claim the database: %w", err)
|
||||||
}
|
}
|
||||||
defer holdServe()
|
defer holdServe()
|
||||||
if err := pg.Migrate(ctx, log, cfg.Database.OwnerURL, cfg.Database.URL, schemas); err != nil {
|
// Nothing is served yet, so there is no readiness to report while this retries.
|
||||||
|
migrate := func(ctx context.Context) error {
|
||||||
|
return pg.Migrate(ctx, log, cfg.Database.OwnerURL, cfg.Database.URL, schemas)
|
||||||
|
}
|
||||||
|
if err := retryStartup(ctx, log, func(string) {}, stageMigrate, migrate); err != nil {
|
||||||
_ = db.Close()
|
_ = db.Close()
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
+42
-10
@@ -15,7 +15,6 @@ import (
|
|||||||
_ "github.com/jackc/pgx/v5/stdlib"
|
_ "github.com/jackc/pgx/v5/stdlib"
|
||||||
"github.com/jmoiron/sqlx"
|
"github.com/jmoiron/sqlx"
|
||||||
"github.com/pressly/goose/v3"
|
"github.com/pressly/goose/v3"
|
||||||
"github.com/pressly/goose/v3/lock"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
//go:embed migrations
|
//go:embed migrations
|
||||||
@@ -34,10 +33,17 @@ func Open(ctx context.Context, url string) (*sqlx.DB, error) {
|
|||||||
return db, nil
|
return db, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Migrate applies every pending migration as the owner, under a session
|
// migrateLockName names the advisory lock that serialises migration and
|
||||||
// advisory lock, so two instances starting together never race, then grants
|
// privilege grants across instances.
|
||||||
// the runtime role DML on each module schema. A failed migration returns an
|
const migrateLockName = "migrate"
|
||||||
// error and the boot stops.
|
|
||||||
|
// Migrate applies every pending migration as the owner, then grants the
|
||||||
|
// runtime role DML on each module schema. Both steps run under one session
|
||||||
|
// advisory lock, so two instances starting together take turns: the second
|
||||||
|
// finds the migrations applied and the grants already in place. Granting on the
|
||||||
|
// same object from two sessions fails with "tuple concurrently updated", so the
|
||||||
|
// grants need the lock as much as the migrations do. It returns an error on any
|
||||||
|
// failure, and the caller retries.
|
||||||
func Migrate(ctx context.Context, log *slog.Logger, ownerURL, runtimeURL string, schemas []string) error {
|
func Migrate(ctx context.Context, log *slog.Logger, ownerURL, runtimeURL string, schemas []string) error {
|
||||||
db, err := Open(ctx, ownerURL)
|
db, err := Open(ctx, ownerURL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -45,15 +51,17 @@ func Migrate(ctx context.Context, log *slog.Logger, ownerURL, runtimeURL string,
|
|||||||
}
|
}
|
||||||
defer func() { _ = db.Close() }()
|
defer func() { _ = db.Close() }()
|
||||||
|
|
||||||
|
release, err := lockMigrate(ctx, db)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer release()
|
||||||
|
|
||||||
files, err := fs.Sub(migrationFiles, "migrations")
|
files, err := fs.Sub(migrationFiles, "migrations")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("migration files: %w", err)
|
return fmt.Errorf("migration files: %w", err)
|
||||||
}
|
}
|
||||||
locker, err := lock.NewPostgresSessionLocker()
|
provider, err := goose.NewProvider(goose.DialectPostgres, db.DB, files)
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("migration lock: %w", err)
|
|
||||||
}
|
|
||||||
provider, err := goose.NewProvider(goose.DialectPostgres, db.DB, files, goose.WithSessionLocker(locker))
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("migration provider: %w", err)
|
return fmt.Errorf("migration provider: %w", err)
|
||||||
}
|
}
|
||||||
@@ -67,6 +75,30 @@ func Migrate(ctx context.Context, log *slog.Logger, ownerURL, runtimeURL string,
|
|||||||
return grantRuntime(ctx, db, runtimeURL, schemas)
|
return grantRuntime(ctx, db, runtimeURL, schemas)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// lockMigrate waits for the migration advisory lock and holds it on a dedicated
|
||||||
|
// connection until release is called. Goose's own session locker covers only
|
||||||
|
// the migrations, which leaves the grants unprotected, so the lock is taken here
|
||||||
|
// around both.
|
||||||
|
func lockMigrate(ctx context.Context, db *sqlx.DB) (release func(), err error) {
|
||||||
|
conn, err := db.Conn(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("lock migrations: %w", err)
|
||||||
|
}
|
||||||
|
if _, err := conn.ExecContext(ctx, "SELECT pg_advisory_lock(hashtextextended($1, 0))", migrateLockKey()); err != nil {
|
||||||
|
_ = conn.Close()
|
||||||
|
return nil, fmt.Errorf("lock migrations: %w", err)
|
||||||
|
}
|
||||||
|
return func() {
|
||||||
|
// Closing the connection would release the lock too, but the pool may
|
||||||
|
// keep it open.
|
||||||
|
_, _ = conn.ExecContext(context.WithoutCancel(ctx),
|
||||||
|
"SELECT pg_advisory_unlock(hashtextextended($1, 0))", migrateLockKey())
|
||||||
|
_ = conn.Close()
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func migrateLockKey() string { return "archie." + migrateLockName + ".owner" }
|
||||||
|
|
||||||
// grantRuntime gives the runtime role USAGE and DML on each schema, never DDL.
|
// grantRuntime gives the runtime role USAGE and DML on each schema, never DDL.
|
||||||
func grantRuntime(ctx context.Context, db *sqlx.DB, runtimeURL string, schemas []string) error {
|
func grantRuntime(ctx context.Context, db *sqlx.DB, runtimeURL string, schemas []string) error {
|
||||||
cfg, err := pgx.ParseConfig(runtimeURL)
|
cfg, err := pgx.ParseConfig(runtimeURL)
|
||||||
|
|||||||
Reference in new issue
Block a user