From 651021130df21c07e7955ad6e1ddc0a2e15bcd23 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Thu, 24 Sep 2026 09:32:39 +1000 Subject: [PATCH 1/2] feat(postgres): whole-database backup and offline restore on pg_dump/pg_restore Restore claims every service role (state-store and gateway) non-blockingly and refuses if any is held or the claim cannot be checked; it drops tables a later migration added before pg_restore --clean so a restored schema matches the snapshot. Backup validates the archive before replacing the previous one. Ports the store's write-side clip, park-class and transition-audit coverage from internal/store. --- internal/app/archied/state_store_pool.go | 2 +- .../infrastructure/postgres/maintenance.go | 264 +++++++++++++++ .../postgres/maintenance_test.go | 302 ++++++++++++++++++ .../postgres/normalisation_test.go | 216 +++++++++++++ 4 files changed, 783 insertions(+), 1 deletion(-) create mode 100644 internal/infrastructure/postgres/maintenance.go create mode 100644 internal/infrastructure/postgres/maintenance_test.go create mode 100644 internal/infrastructure/postgres/normalisation_test.go diff --git a/internal/app/archied/state_store_pool.go b/internal/app/archied/state_store_pool.go index e54959173..5f51809d4 100644 --- a/internal/app/archied/state_store_pool.go +++ b/internal/app/archied/state_store_pool.go @@ -35,7 +35,7 @@ func (b *boot) openStateStorePool(ctx context.Context) error { b.log.Error("migrate state store postgres schema", "err", err) return err } - ownership, err := postgres.AcquireOwnership(ctx, pool, "state-store") + ownership, err := postgres.AcquireOwnership(ctx, pool, postgres.OwnerStateStore) if err != nil { pool.Close() b.log.Error("claim state store ownership", "err", err) diff --git a/internal/infrastructure/postgres/maintenance.go b/internal/infrastructure/postgres/maintenance.go new file mode 100644 index 000000000..90bf84ca9 --- /dev/null +++ b/internal/infrastructure/postgres/maintenance.go @@ -0,0 +1,264 @@ +package postgres + +import ( + "bytes" + "context" + "errors" + "fmt" + "io/fs" + "net/url" + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/pressly/goose/v3" +) + +// The serving roles that each hold an ownership claim for their whole life. +const ( + OwnerStateStore = "state-store" + OwnerGateway = "gateway" +) + +// ServiceOwners is every serving role that reads or writes the database. A +// destructive offline operation must hold all of them: one database backs +// every service, so the State Store being stopped alone proves nothing about +// the Gateway. +var ServiceOwners = []string{OwnerStateStore, OwnerGateway} + +// ErrSchemaTooNew is returned for a database migrated by a newer release than +// this binary. goose Down is not a rollback (it drops tables), so the only way +// back is restoring the snapshot taken before that migration. +var ErrSchemaTooNew = errors.New("postgres: database schema is newer than this binary supports") + +// gooseTable is goose's version table; its presence is what marks an archive +// as a dump of an archie database rather than of anything pg_restore can read. +const gooseTable = "public.goose_db_version" + +// SupportedSchemaVersion is the highest migration this binary embeds. +func SupportedSchemaVersion() int64 { + names, err := fs.Glob(Migrations(), "*.sql") + if err != nil { + // Unreachable: the pattern is well formed. + panic(err) + } + var highest int64 + for _, name := range names { + if version, err := goose.NumericComponent(name); err == nil { + highest = max(highest, version) + } + } + return highest +} + +// CheckSchemaVersion refuses a schema this binary cannot serve. +func CheckSchemaVersion(version int64) error { + if supported := SupportedSchemaVersion(); version > supported { + return fmt.Errorf("%w (database at %d, binary supports %d)", ErrSchemaTooNew, version, supported) + } + return nil +} + +// SchemaVersion reads the applied goose version without migrating. An +// unmigrated database is version 0. +func SchemaVersion(ctx context.Context, pool *pgxpool.Pool) (int64, error) { + var version int64 + err := pool.QueryRow(ctx, `SELECT CASE WHEN to_regclass('public.goose_db_version') IS NULL THEN 0 + ELSE (SELECT coalesce(max(version_id), 0) FROM public.goose_db_version WHERE is_applied) END`).Scan(&version) + if err != nil { + return 0, fmt.Errorf("postgres: read schema version: %w", err) + } + return version, nil +} + +// AcquireAll claims every name, or none: on any failure the claims already +// taken are released before the error returns. +func AcquireAll(ctx context.Context, pool *pgxpool.Pool, names []string) (release func(context.Context), err error) { + held := make([]*Ownership, 0, len(names)) + release = func(ctx context.Context) { + for _, o := range slices.Backward(held) { + _ = o.Release(ctx) + } + } + for _, name := range names { + o, err := AcquireOwnership(ctx, pool, name) + if err != nil { + release(ctx) + return nil, err + } + held = append(held, o) + } + return release, nil +} + +// Backup writes a whole-database pg_dump archive to out. It takes no claim: +// pg_dump reads one consistent snapshot, so it is safe against serving +// processes, which is what the update installer needs. The archive goes to a +// scratch file and replaces out only once pg_restore can read it back, so a +// failed run never costs the previous good snapshot. +func Backup(ctx context.Context, databaseURL, out string) error { + tmp, err := os.CreateTemp(filepath.Dir(out), filepath.Base(out)+".tmp*") + if err != nil { + return fmt.Errorf("postgres: backup: %w", err) + } + scratch := tmp.Name() + _ = tmp.Close() + defer func() { _ = os.Remove(scratch) }() + + dbname, env := libpqTarget(databaseURL) + if _, err := runTool(ctx, env, "pg_dump", dumpArgs(dbname, scratch)...); err != nil { + return fmt.Errorf("postgres: backup: %w", err) + } + if _, err := ValidateSnapshot(ctx, scratch); err != nil { + return fmt.Errorf("postgres: backup produced an unreadable snapshot: %w", err) + } + if err := os.Rename(scratch, out); err != nil { + return fmt.Errorf("postgres: backup: %w", err) + } + return nil +} + +// ValidateSnapshot reads the archive's table of contents and returns the +// tables it holds. An archive without goose's version table is not a dump of +// an archie database. +func ValidateSnapshot(ctx context.Context, path string) ([]string, error) { + if _, err := os.Stat(path); err != nil { + return nil, fmt.Errorf("postgres: snapshot: %w", err) + } + list, err := runTool(ctx, nil, "pg_restore", "--list", path) + if err != nil { + return nil, fmt.Errorf("postgres: snapshot %s: %w", path, err) + } + tables := snapshotTables(list) + if !slices.Contains(tables, gooseTable) { + return nil, fmt.Errorf("postgres: snapshot %s holds no %s; it is not an archie database dump", path, gooseTable) + } + return tables, nil +} + +// Restore replaces the whole database with the snapshot at from. It is +// destructive: every write after the snapshot is lost. It refuses unless it +// can claim every service role, so no serving process is running and none can +// start until it finishes; a claim that cannot be checked is a refusal too. +// +// pg_restore --clean drops only what the archive holds, so tables a later +// migration created are dropped first; otherwise they would survive into the +// restored schema and break the next upgrade's migration. +func Restore(ctx context.Context, databaseURL, from string) error { + tables, err := ValidateSnapshot(ctx, from) + if err != nil { + return err + } + pool, err := Open(ctx, databaseURL) + if err != nil { + return fmt.Errorf("postgres: restore cannot confirm every service is stopped: %w", err) + } + defer pool.Close() + release, err := AcquireAll(ctx, pool, ServiceOwners) + if err != nil { + return fmt.Errorf("postgres: restore refused, stop every archie service first: %w", err) + } + defer release(context.WithoutCancel(ctx)) + + if err := dropTablesNotIn(ctx, pool, tables); err != nil { + return err + } + dbname, env := libpqTarget(databaseURL) + if _, err := runTool(ctx, env, "pg_restore", restoreArgs(dbname, from)...); err != nil { + return fmt.Errorf("postgres: restore: %w", err) + } + return nil +} + +func dropTablesNotIn(ctx context.Context, pool *pgxpool.Pool, keep []string) error { + rows, err := pool.Query(ctx, `SELECT schemaname, tablename FROM pg_tables + WHERE schemaname NOT IN ('pg_catalog', 'information_schema')`) + if err != nil { + return fmt.Errorf("postgres: restore: list tables: %w", err) + } + var drop []pgx.Identifier + for rows.Next() { + var schema, table string + if err := rows.Scan(&schema, &table); err != nil { + rows.Close() + return fmt.Errorf("postgres: restore: list tables: %w", err) + } + if !slices.Contains(keep, schema+"."+table) { + drop = append(drop, pgx.Identifier{schema, table}) + } + } + rows.Close() + if err := rows.Err(); err != nil { + return fmt.Errorf("postgres: restore: list tables: %w", err) + } + for _, table := range drop { + if _, err := pool.Exec(ctx, "DROP TABLE IF EXISTS "+table.Sanitize()+" CASCADE"); err != nil { + return fmt.Errorf("postgres: restore: drop %s: %w", table.Sanitize(), err) + } + } + return nil +} + +// snapshotTables extracts schema.table for every TABLE entry of a +// `pg_restore --list` listing, sorted. +func snapshotTables(list string) []string { + var tables []string + for line := range strings.Lines(list) { + if strings.HasPrefix(line, ";") { + continue + } + _, entry, ok := strings.Cut(line, ";") + if !ok { + continue + } + fields := strings.Fields(entry) + // "tableoid oid TABLE schema name owner"; "TABLE DATA" entries are rows. + if len(fields) >= 5 && fields[2] == "TABLE" && fields[3] != "DATA" { + tables = append(tables, fields[3]+"."+fields[4]) + } + } + slices.Sort(tables) + return tables +} + +func dumpArgs(dbname, out string) []string { + return []string{"--format=custom", "--no-password", "--file=" + out, "--dbname=" + dbname} +} + +func restoreArgs(dbname, from string) []string { + return []string{"--clean", "--if-exists", "--single-transaction", "--exit-on-error", "--no-owner", "--no-password", "--dbname=" + dbname, from} +} + +// libpqTarget splits a connection URL into the dbname argument and the +// environment the client tools read, moving the password to PGPASSWORD so it +// never appears in a process listing. A key=value connection string is passed +// through as-is. +func libpqTarget(databaseURL string) (string, []string) { + u, err := url.Parse(databaseURL) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") || u.User == nil { + return databaseURL, nil + } + password, ok := u.User.Password() + if !ok { + return databaseURL, nil + } + u.User = url.User(u.User.Username()) + return u.String(), []string{"PGPASSWORD=" + password} +} + +// runTool runs a PostgreSQL client tool and returns its stdout; a failure +// carries its stderr, which is where the tools explain themselves. +func runTool(ctx context.Context, env []string, name string, args ...string) (string, error) { + cmd := exec.CommandContext(ctx, name, args...) + cmd.Env = append(os.Environ(), env...) + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("%s: %w: %s", name, err, strings.TrimSpace(stderr.String())) + } + return stdout.String(), nil +} diff --git a/internal/infrastructure/postgres/maintenance_test.go b/internal/infrastructure/postgres/maintenance_test.go new file mode 100644 index 000000000..6c1244fa7 --- /dev/null +++ b/internal/infrastructure/postgres/maintenance_test.go @@ -0,0 +1,302 @@ +package postgres + +import ( + "errors" + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/samcharles93/archie-core/internal/infrastructure/postgres/pgtest" +) + +func TestLibpqTargetKeepsThePasswordOffTheCommandLine(t *testing.T) { + tests := []struct { + name string + url string + wantDB string + wantEnv []string + leakFree string + }{ + { + name: "url with password", + url: "postgres://archie:s3cret@db.example:5432/archie?sslmode=require", + wantDB: "postgres://archie@db.example:5432/archie?sslmode=require", + wantEnv: []string{"PGPASSWORD=s3cret"}, + leakFree: "s3cret", + }, + { + name: "url without password", + url: "postgresql://archie@db.example/archie", + wantDB: "postgresql://archie@db.example/archie", + }, + { + name: "key value connection string passes through", + url: "host=db.example dbname=archie", + wantDB: "host=db.example dbname=archie", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + db, env := libpqTarget(tt.url) + if db != tt.wantDB { + t.Errorf("dbname = %q, want %q", db, tt.wantDB) + } + if !slices.Equal(env, tt.wantEnv) { + t.Errorf("env = %q, want %q", env, tt.wantEnv) + } + if tt.leakFree != "" && strings.Contains(db, tt.leakFree) { + t.Errorf("dbname %q carries the password", db) + } + }) + } +} + +func TestToolArguments(t *testing.T) { + tests := []struct { + name string + got []string + want []string + }{ + { + name: "dump is a whole-database custom-format archive", + got: dumpArgs("postgres://h/db", "/tmp/out.dump"), + want: []string{"--format=custom", "--no-password", "--file=/tmp/out.dump", "--dbname=postgres://h/db"}, + }, + { + name: "restore cleans, stops on the first error and is atomic", + got: restoreArgs("postgres://h/db", "/tmp/in.dump"), + want: []string{"--clean", "--if-exists", "--single-transaction", "--exit-on-error", "--no-owner", "--no-password", "--dbname=postgres://h/db", "/tmp/in.dump"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if !slices.Equal(tt.got, tt.want) { + t.Errorf("args = %q, want %q", tt.got, tt.want) + } + }) + } +} + +func TestSnapshotTablesReadsTheArchiveList(t *testing.T) { + list := `; +; Archive created at 2026-09-24 10:00:00 UTC +; +215; 1259 16390 TABLE public tasks archie +216; 1259 16391 SEQUENCE public tasks_id_seq archie +3401; 0 16390 TABLE DATA public tasks archie +217; 1259 16400 TABLE public goose_db_version archie +` + got := snapshotTables(list) + want := []string{"public.goose_db_version", "public.tasks"} + if !slices.Equal(got, want) { + t.Fatalf("snapshotTables = %q, want %q", got, want) + } +} + +func requireTools(t *testing.T) { + t.Helper() + for _, tool := range []string{"pg_dump", "pg_restore"} { + if _, err := exec.LookPath(tool); err != nil { + t.Skipf("%s is not on PATH; the integration test needs the PostgreSQL 18 client tools", tool) + } + } +} + +// migratedURL is a migrated database and a pool on it. +func migratedURL(t *testing.T) (string, *pgxpool.Pool) { + t.Helper() + url := pgtest.URL(t) + pool := openPool(t, url) + if err := Migrate(t.Context(), pool, Migrations()); err != nil { + t.Fatalf("Migrate: %v", err) + } + return url, pool +} + +func titles(t *testing.T, pool *pgxpool.Pool) []string { + t.Helper() + rows, err := pool.Query(t.Context(), "SELECT title FROM tasks ORDER BY id") + if err != nil { + t.Fatalf("list tasks: %v", err) + } + defer rows.Close() + var out []string + for rows.Next() { + var title string + if err := rows.Scan(&title); err != nil { + t.Fatal(err) + } + out = append(out, title) + } + return out +} + +func enqueue(t *testing.T, pool *pgxpool.Pool, title string) { + t.Helper() + if _, err := New(pool).EnqueueChatTask(t.Context(), "acme", "widget", title, "body", "implement", ""); err != nil { + t.Fatalf("enqueue %q: %v", title, err) + } +} + +// A restore after a forward migration is the case restore exists for: the +// tables the later migration added are not in the snapshot, so --clean alone +// would leave them behind for the next upgrade to trip over. +func TestBackupThenRestoreReturnsTheDatabaseToTheSnapshot(t *testing.T) { + requireTools(t) + url, pool := migratedURL(t) + enqueue(t, pool, "before") + before, err := SchemaVersion(t.Context(), pool) + if err != nil { + t.Fatalf("SchemaVersion: %v", err) + } + snapshot := filepath.Join(t.TempDir(), "snapshot.dump") + if err := Backup(t.Context(), url, snapshot); err != nil { + t.Fatalf("Backup: %v", err) + } + if leftovers, _ := filepath.Glob(snapshot + ".tmp*"); len(leftovers) != 0 { + t.Errorf("backup left scratch files: %v", leftovers) + } + + enqueue(t, pool, "after") + if _, err := pool.Exec(t.Context(), "CREATE TABLE later_migration (id bigint)"); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(t.Context(), "INSERT INTO goose_db_version (version_id, is_applied) VALUES ($1, true)", before+1); err != nil { + t.Fatal(err) + } + + if err := Restore(t.Context(), url, snapshot); err != nil { + t.Fatalf("Restore: %v", err) + } + if got := titles(t, pool); !slices.Equal(got, []string{"before"}) { + t.Errorf("tasks after restore = %q, want only the snapshot's", got) + } + var extra bool + if err := pool.QueryRow(t.Context(), "SELECT to_regclass('public.later_migration') IS NOT NULL").Scan(&extra); err != nil { + t.Fatal(err) + } + if extra { + t.Error("restore left a table the snapshot does not contain") + } + if after, err := SchemaVersion(t.Context(), pool); err != nil || after != before { + t.Errorf("schema version after restore = %d, %v; want %d", after, err, before) + } + enqueue(t, pool, "writable") +} + +func TestRestoreRefusesWhileAnyServiceIsServing(t *testing.T) { + requireTools(t) + for _, owner := range ServiceOwners { + t.Run(owner, func(t *testing.T) { + url, pool := migratedURL(t) + enqueue(t, pool, "snapshot") + snapshot := filepath.Join(t.TempDir(), "snapshot.dump") + if err := Backup(t.Context(), url, snapshot); err != nil { + t.Fatalf("Backup: %v", err) + } + enqueue(t, pool, "live") + claimOther(t, url, owner, false, false) + + err := Restore(t.Context(), url, snapshot) + if !errors.Is(err, ErrOwned) { + t.Fatalf("Restore with %s serving = %v, want ErrOwned", owner, err) + } + if !strings.Contains(err.Error(), owner) { + t.Errorf("refusal must name the serving role %q: %v", owner, err) + } + if got := titles(t, pool); !slices.Equal(got, []string{"snapshot", "live"}) { + t.Errorf("refused restore changed the database: %q", got) + } + // Every claim the refused restore took is dropped again. + for _, other := range ServiceOwners { + if other == owner { + continue + } + claim, err := AcquireOwnership(t.Context(), pool, other) + if err != nil { + t.Fatalf("%s still held after a refused restore: %v", other, err) + } + _ = claim.Release(t.Context()) + } + }) + } +} + +func TestRestoreRefusesWhatItCannotCheck(t *testing.T) { + requireTools(t) + url, pool := migratedURL(t) + snapshot := filepath.Join(t.TempDir(), "snapshot.dump") + if err := Backup(t.Context(), url, snapshot); err != nil { + t.Fatalf("Backup: %v", err) + } + notASnapshot := filepath.Join(t.TempDir(), "garbage.dump") + if err := os.WriteFile(notASnapshot, []byte("not an archive"), 0o600); err != nil { + t.Fatal(err) + } + enqueue(t, pool, "live") + + tests := []struct { + name string + url string + from string + }{ + {name: "unreachable server", url: "postgres://archie:archie@127.0.0.1:1/none?connect_timeout=2", from: snapshot}, + {name: "not a snapshot", url: url, from: notASnapshot}, + {name: "missing snapshot", url: url, from: filepath.Join(t.TempDir(), "absent.dump")}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := Restore(t.Context(), tt.url, tt.from); err == nil { + t.Fatal("Restore succeeded, want refusal") + } + if got := titles(t, pool); !slices.Equal(got, []string{"live"}) { + t.Errorf("refused restore changed the database: %q", got) + } + }) + } +} + +func TestBackupFailureKeepsThePreviousSnapshot(t *testing.T) { + requireTools(t) + out := filepath.Join(t.TempDir(), "snapshot.dump") + if err := os.WriteFile(out, []byte("good snapshot"), 0o600); err != nil { + t.Fatal(err) + } + if err := Backup(t.Context(), "postgres://archie:archie@127.0.0.1:1/none?connect_timeout=2", out); err == nil { + t.Fatal("Backup of an unreachable database succeeded") + } + if data, _ := os.ReadFile(out); string(data) != "good snapshot" { + t.Errorf("failed backup replaced the previous snapshot: %q", data) + } + if leftovers, _ := filepath.Glob(out + ".tmp*"); len(leftovers) != 0 { + t.Errorf("failed backup left scratch files: %v", leftovers) + } +} + +func TestCheckSchemaVersion(t *testing.T) { + supported := SupportedSchemaVersion() + if supported < 4 { + t.Fatalf("SupportedSchemaVersion = %d, want the highest embedded migration", supported) + } + tests := []struct { + name string + version int64 + wantErr error + }{ + {name: "current", version: supported}, + {name: "older is upgradable", version: 1}, + {name: "newer than this binary", version: supported + 1, wantErr: ErrSchemaTooNew}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := CheckSchemaVersion(tt.version); !errors.Is(err, tt.wantErr) { + t.Fatalf("CheckSchemaVersion(%d) = %v, want %v", tt.version, err, tt.wantErr) + } + }) + } +} diff --git a/internal/infrastructure/postgres/normalisation_test.go b/internal/infrastructure/postgres/normalisation_test.go new file mode 100644 index 000000000..7609c8c0b --- /dev/null +++ b/internal/infrastructure/postgres/normalisation_test.go @@ -0,0 +1,216 @@ +package postgres + +import ( + "context" + "strings" + "testing" + "unicode/utf8" + + "github.com/samcharles93/archie-core/internal/domain/workflow" + "github.com/samcharles93/archie-core/internal/events" + "github.com/samcharles93/archie-core/internal/infrastructure/postgres/pgtest" +) + +func storeFor(t *testing.T) *Store { + t.Helper() + pool := openPool(t, pgtest.URL(t)) + if err := Migrate(t.Context(), pool, Migrations()); err != nil { + t.Fatalf("Migrate: %v", err) + } + return New(pool) +} + +// runningTask enqueues one issue and claims it, the state every park starts in. +func runningTask(t *testing.T, s *Store) *workflow.Task { + t.Helper() + if _, err := s.EnqueueIssue(t.Context(), "acme", "widgets", 1, "t", "b", "", ""); err != nil { + t.Fatal(err) + } + task, err := s.ClaimNext(t.Context()) + if err != nil || task == nil { + t.Fatalf("ClaimNext = %v, %v", task, err) + } + return task +} + +type transitionRow struct{ from, to, detail string } + +func transitions(t *testing.T, s *Store, taskID int64) []transitionRow { + t.Helper() + rows, err := s.pool.Query(t.Context(), "SELECT from_status, to_status, detail FROM transitions WHERE task_id = $1 ORDER BY id", taskID) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var out []transitionRow + for rows.Next() { + var r transitionRow + if err := rows.Scan(&r.from, &r.to, &r.detail); err != nil { + t.Fatal(err) + } + out = append(out, r) + } + return out +} + +// Every free-text column the SQLite store clipped to 4000 bytes on a rune +// boundary is clipped the same way here. The oversized value ends in a +// multi-byte rune straddling the limit, so a byte cut would store invalid +// UTF-8 and a missing clip would store all of it. +func TestWritesClipFreeTextTo4000Bytes(t *testing.T) { + oversized := strings.Repeat("x", 3999) + "é" + strings.Repeat("y", 1000) + want := strings.Repeat("x", 3999) + + tests := []struct { + name string + write func(t *testing.T, s *Store) string + }{ + {name: "transition to parked: park reason and transition detail", write: func(t *testing.T, s *Store) string { + task := runningTask(t, s) + if err := s.Transition(t.Context(), task.ID, workflow.StatusRunning, workflow.StatusParked, oversized); err != nil { + t.Fatal(err) + } + rows := transitions(t, s, task.ID) + if len(rows) == 0 || rows[len(rows)-1].detail != want { + t.Errorf("transition detail not clipped: %d rows", len(rows)) + } + got, _ := s.TaskByID(t.Context(), task.ID) + return got.ParkReason + }}, + {name: "park task: park reason and transition detail", write: func(t *testing.T, s *Store) string { + task := runningTask(t, s) + if err := s.ParkTask(t.Context(), task.ID, workflow.StatusRunning, oversized, "transient"); err != nil { + t.Fatal(err) + } + rows := transitions(t, s, task.ID) + if len(rows) == 0 || rows[len(rows)-1].detail != want { + t.Errorf("transition detail not clipped: %d rows", len(rows)) + } + got, _ := s.TaskByID(t.Context(), task.ID) + return got.ParkReason + }}, + {name: "update: park reason", write: func(t *testing.T, s *Store) string { + task := runningTask(t, s) + task.ParkReason = oversized + if err := s.Update(t.Context(), task); err != nil { + t.Fatal(err) + } + got, _ := s.TaskByID(t.Context(), task.ID) + return got.ParkReason + }}, + {name: "begin remediation: review payload", write: func(t *testing.T, s *Store) string { + task := runningTask(t, s) + if err := s.Transition(t.Context(), task.ID, workflow.StatusRunning, workflow.StatusPROpen, "pr"); err != nil { + t.Fatal(err) + } + if err := s.BeginRemediation(t.Context(), task.ID, oversized); err != nil { + t.Fatal(err) + } + got, _ := s.TaskByID(t.Context(), task.ID) + return got.ReviewPayload + }}, + {name: "update review payload", write: func(t *testing.T, s *Store) string { + task := runningTask(t, s) + if err := s.Transition(t.Context(), task.ID, workflow.StatusRunning, workflow.StatusPROpen, "pr"); err != nil { + t.Fatal(err) + } + if err := s.BeginRemediation(t.Context(), task.ID, "first"); err != nil { + t.Fatal(err) + } + if err := s.UpdateReviewPayload(t.Context(), task.ID, oversized); err != nil { + t.Fatal(err) + } + got, _ := s.TaskByID(t.Context(), task.ID) + return got.ReviewPayload + }}, + {name: "event detail", write: func(t *testing.T, s *Store) string { + if _, err := s.InsertEvent(t.Context(), events.Event{Kind: events.KindLog, Detail: oversized}); err != nil { + t.Fatal(err) + } + got, err := s.EventsSince(t.Context(), "", 1) + if err != nil || len(got) != 1 { + t.Fatalf("EventsSince = %d, %v", len(got), err) + } + return got[0].Detail + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.write(t, storeFor(t)) + if got != want || !utf8.ValidString(got) { + t.Errorf("stored %d bytes (valid UTF-8 %v), want the %d-byte rune-boundary clip", len(got), utf8.ValidString(got), len(want)) + } + }) + } +} + +func TestParkClassNormalisation(t *testing.T) { + tests := []struct { + name string + park func(ctx context.Context, s *Store, id int64) error + want string + }{ + {name: "generic transition to parked is needs_human", want: "needs_human", park: func(ctx context.Context, s *Store, id int64) error { + return s.Transition(ctx, id, workflow.StatusRunning, workflow.StatusParked, "gate failed") + }}, + {name: "known class is kept", want: "transient", park: func(ctx context.Context, s *Store, id int64) error { + return s.ParkTask(ctx, id, workflow.StatusRunning, "container acquire failed", "transient") + }}, + {name: "unknown class falls back to needs_human", want: "needs_human", park: func(ctx context.Context, s *Store, id int64) error { + return s.ParkTask(ctx, id, workflow.StatusRunning, "x", "urgent") + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := storeFor(t) + task := runningTask(t, s) + if err := tt.park(t.Context(), s, task.ID); err != nil { + t.Fatal(err) + } + got, _ := s.TaskByID(t.Context(), task.ID) + if got.Status != workflow.StatusParked || got.ParkClass != tt.want { + t.Fatalf("status %q class %q, want parked %q", got.Status, got.ParkClass, tt.want) + } + // Requeue clears class and reason together: a stale class on a + // queued task would group a live task with parked ones. + if err := s.Requeue(t.Context(), task.ID, workflow.StatusParked, "implement"); err != nil { + t.Fatal(err) + } + got, _ = s.TaskByID(t.Context(), task.ID) + if got.ParkClass != "needs_human" || got.ParkReason != "" { + t.Errorf("after requeue: class %q reason %q, want cleared", got.ParkClass, got.ParkReason) + } + }) + } +} + +// A transition that leaves parked alone keeps the previous park fields, and +// every guarded write leaves exactly one audit row in order. +func TestTransitionsRecordEachMoveInOrder(t *testing.T) { + s := storeFor(t) + task := runningTask(t, s) + ctx := t.Context() + if err := s.ParkTask(ctx, task.ID, workflow.StatusRunning, "why", "transient"); err != nil { + t.Fatal(err) + } + if err := s.Requeue(ctx, task.ID, workflow.StatusParked, "implement"); err != nil { + t.Fatal(err) + } + // A stale transition writes no audit row. + if err := s.Transition(ctx, task.ID, workflow.StatusRunning, workflow.StatusPROpen, "stale"); err == nil { + t.Fatal("stale Transition succeeded") + } + got := transitions(t, s, task.ID) + want := []transitionRow{ + {workflow.StatusRunning, workflow.StatusParked, "why"}, + {workflow.StatusParked, workflow.StatusQueued, "requeued implement"}, + } + if len(got) != len(want) { + t.Fatalf("transitions = %+v, want %+v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("transition %d = %+v, want %+v", i, got[i], want[i]) + } + } +} From 1b5ba30113f93ea2f88f7d27d2a471090329b473 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Thu, 24 Sep 2026 09:50:10 +1000 Subject: [PATCH 2/2] feat(archied): recovery commands on PostgreSQL; fail-closed pre-update snapshot, no automatic schema rollback Without -db, backup/restore/validate/rollback resolve database_url from the config and work on the whole PostgreSQL database. Restore claims every service role; rollback claims the State Store role; validate refuses a schema newer than the binary. The installer refuses a daemon update it cannot snapshot first; the watchdog never restores the database and prints the explicit offline restore instead. --- cmd/archie-state-store/recovery.go | 42 ++-- .../recovery_postgres_test.go | 29 +++ docs/architecture/safe-change-and-recovery.md | 25 +++ docs/data/generated/dev-docs.json | 2 +- internal/app/archied/state_store_recovery.go | 79 ++++--- .../archied/state_store_recovery_postgres.go | 108 ++++++++++ .../state_store_recovery_postgres_test.go | 201 ++++++++++++++++++ .../releaseupdate/db_snapshot_script_test.go | 114 ++++++++++ internal/releaseupdate/install_script_test.go | 2 + scripts/archie-update-install | 19 ++ scripts/archie-update-watchdog | 15 ++ 11 files changed, 588 insertions(+), 48 deletions(-) create mode 100644 cmd/archie-state-store/recovery_postgres_test.go create mode 100644 internal/app/archied/state_store_recovery_postgres.go create mode 100644 internal/app/archied/state_store_recovery_postgres_test.go create mode 100644 internal/releaseupdate/db_snapshot_script_test.go diff --git a/cmd/archie-state-store/recovery.go b/cmd/archie-state-store/recovery.go index 4d115a338..89ce32420 100644 --- a/cmd/archie-state-store/recovery.go +++ b/cmd/archie-state-store/recovery.go @@ -19,18 +19,28 @@ const recoveryUsage = `usage: archie-state-store [flags] With no command, archie-state-store serves the State Store gRPC contract. -The recovery commands operate on the task database file directly -- the -configured db_path with "-tasks.sqlite" appended -- and are how an operator -recovers a database the control plane will not start on. Stop the State Store -first: restore and rollback rewrite the file and refuse while another process -owns it, while backup takes a consistent snapshot of a serving store. +The recovery commands are how an operator recovers a database the control +plane will not start on. Without -db they operate on the PostgreSQL database +the configuration's database_url names, through pg_dump and pg_restore (which +must be on PATH); with -db, on a legacy SQLite task file (the configured +db_path with "-tasks.sqlite" appended). - backup -db FILE -out FILE write a snapshot of the database - restore -db FILE -from FILE replace the database with a snapshot - validate -db FILE [-config FILE] check the file, its stored settings and the - configuration the daemon would boot with - rollback -db FILE -kind KIND replay an earlier revision of a stored +backup takes a consistent snapshot of a serving database. restore is +destructive and offline: stop every archie service first, since it refuses +while any of them holds its ownership claim, and every write made after the +snapshot is lost. rollback refuses while the State Store serves. There is no +automatic schema rollback: the way back from a migration is restoring the +snapshot taken before it. + + backup [-db FILE] -out FILE write a snapshot of the database + restore [-db FILE] -from FILE replace the database with a snapshot + validate [-db FILE] check the database, its stored settings and + the configuration the daemon would boot with + rollback [-db FILE] -kind KIND replay an earlier revision of a stored [-revision N] resource through the ordinary replace + +Every command takes -config FILE and -config-overlay FILE: the configuration +the daemon boots with, which names the database. ` // runRecovery parses and performs one offline recovery command, reporting the @@ -54,17 +64,15 @@ func runRecovery(args []string, stdout, stderr io.Writer) int { flags := flag.NewFlagSet("archie-state-store "+command, flag.ContinueOnError) flags.SetOutput(stderr) flags.Usage = func() { fmt.Fprint(stderr, recoveryUsage) } - flags.StringVar(&options.DB, "db", "", "task database file the State Store owns (-tasks.sqlite)") + flags.StringVar(&options.DB, "db", "", "legacy SQLite task file (-tasks.sqlite); omit to use the configured database_url") flags.StringVar(&options.Out, "out", "", "snapshot file backup writes") flags.StringVar(&options.From, "from", "", "snapshot file restore reads") flags.StringVar(&options.Kind, "kind", "", "control-plane resource kind rollback replays") flags.Int64Var(&options.Revision, "revision", 0, "revision rollback replays (default: the newest one older than the current value)") - // Only validate asks a question about the process rather than the file, so - // only validate takes the configuration the process would boot with. - if command == archied.RecoveryValidate { - flags.StringVar(&options.Config, "config", configuration.DefaultConfigPath(), "configuration file or directory the daemon boots with") - flags.StringVar(&options.Overlay, "config-overlay", "", "configuration overlay file or directory") - } + // The configuration names the PostgreSQL database, and validate also checks + // the stored settings against it. + flags.StringVar(&options.Config, "config", configuration.DefaultConfigPath(), "configuration file or directory the daemon boots with") + flags.StringVar(&options.Overlay, "config-overlay", "", "configuration overlay file or directory") if err := flags.Parse(args[1:]); err != nil { // A requested help is not a wrong command line: the usage above is the // answer, and the serve path exits 0 for the same request. diff --git a/cmd/archie-state-store/recovery_postgres_test.go b/cmd/archie-state-store/recovery_postgres_test.go new file mode 100644 index 000000000..7bebeb698 --- /dev/null +++ b/cmd/archie-state-store/recovery_postgres_test.go @@ -0,0 +1,29 @@ +package main + +import ( + "path/filepath" + "strings" + "testing" +) + +// Without -db every recovery command targets the configured PostgreSQL +// database, so each one accepts -config and refuses, naming the key, when the +// configuration names no database. +func TestRecoveryWithoutDBUsesTheConfiguredDatabase(t *testing.T) { + dir := t.TempDir() + configPath := writeMinimalConfig(t, dir) + tests := [][]string{ + {"backup", "-out", filepath.Join(dir, "snapshot.dump")}, + {"restore", "-from", filepath.Join(dir, "snapshot.dump")}, + {"validate"}, + {"rollback", "-kind", "settings"}, + } + for _, args := range tests { + t.Run(args[0], func(t *testing.T) { + code, _, stderr := runRecoveryCmd(t, append(args, "-config", configPath)...) + if code != 1 || !strings.Contains(stderr, "database_url") { + t.Fatalf("%s exited %d, stderr %q; want 1 naming database_url", args[0], code, stderr) + } + }) + } +} diff --git a/docs/architecture/safe-change-and-recovery.md b/docs/architecture/safe-change-and-recovery.md index a1bb55e4e..f0fc6a1aa 100644 --- a/docs/architecture/safe-change-and-recovery.md +++ b/docs/architecture/safe-change-and-recovery.md @@ -165,6 +165,31 @@ own `sqlite3` shell, so the installer and an operator run one implementation. blunt one, and is also the way back from a schema migration a release cannot grow out of. +### The same commands on PostgreSQL + +Without `-db`, each command works on the PostgreSQL database the configuration's +`database_url` names (`internal/app/archied/state_store_recovery_postgres.go`, +`internal/infrastructure/postgres/maintenance.go`). One database backs every +service, so the scope is always the whole database. + +- `backup -out F` runs `pg_dump --format=custom`, reads the archive back with + `pg_restore --list` and only then replaces `F`. It takes no claim and runs + against a serving database. +- `restore -from F` is destructive and offline. It claims every service role's + ownership lock (`state-store` and `gateway`) without waiting and refuses if + any is held or the check cannot complete, then drops tables the snapshot does + not hold and runs `pg_restore --clean --single-transaction`. Every write made + after the snapshot is lost. +- `validate` refuses a schema newer than the binary's highest migration, then + runs the same two validation layers. +- `rollback` holds the `state-store` claim for the replay. + +There is no automatic schema rollback. goose `Down` drops tables, so it is data +loss, not a rollback. `scripts/archie-update-install` refuses a daemon update +it cannot snapshot first, and on a failed update `scripts/archie-update-watchdog` +rolls the binaries back, leaves the database alone and prints the `restore` +command for the pre-update snapshot. Whether to run it is the operator's call. + ## Gateway restart: two constraints learned the hard way Telegram `/restart` (`internal/channels/telegram/restart.go` plus the `Start` diff --git a/docs/data/generated/dev-docs.json b/docs/data/generated/dev-docs.json index 8ea3079f8..77e66069a 100644 --- a/docs/data/generated/dev-docs.json +++ b/docs/data/generated/dev-docs.json @@ -100,7 +100,7 @@ "url": "/dev/architecture/safe-change-and-recovery/", "title": "Safe Change and Recovery", "section": "Architecture", - "body": "# Safe Change and Recovery\n\n**Status:** Requirements approved; mechanics are under design. Database-backed settings are implemented through the control plane (2026-09): see the degrade/recovery section below, which records a gap rather than a solution. The full runtime-supervision protocol (bounded observation, versioned audit trail) remains under design.\n**Date:** 2026-07-28 \n**Tracking issue:** [#73](https://github.com/samcharles93/archie-core/issues/73)\n\n## Purpose\n\nArchie is built and improved by agents as well as humans. The architecture MUST\nmake a change easy to locate, understand, test, audit, extend, and reverse\nwithout requiring its author to understand unrelated parts of the application.\n\nSeparation of concerns is therefore an operational safety boundary, not only a\ncode-organisation preference.\n\n## Required outcomes\n\n- An agent can improve a focused feature without receiving authority over\n unrelated domains.\n- Every event, function, action, setting, and policy has an identifiable owner\n and provenance.\n- Source and runtime changes pass deterministic checks outside the actor making\n the change.\n- A rejected or unhealthy candidate cannot replace the last known working\n version.\n- Invalid user configuration does not terminate an otherwise healthy\n application.\n- Configuration and service failures are observable, attributed, and eligible\n for bounded remediation or rollback.\n- The application continues serving unaffected capabilities when one\n capability is misconfigured or unhealthy.\n- Recovery failure is reported and contained rather than hidden in an\n unbounded retry loop.\n\n## Source changes\n\nAgent-authored source changes occur through the task-execution boundary in an\nisolated workspace. The agent does not directly mutate the running\ninstallation.\n\nPromotion is controlled by deterministic repository-owned policy, including\ntests, generated-reference drift checks, protected paths, change scope, and\nother required gates. The actor proposing a change cannot waive these checks.\n\nRepository and domain separation SHOULD allow a task to load and modify only\nthe governing requirements and code for its declared scope.\n\n## Runtime changes\n\nConfiguration and other runtime changes are proposed as candidates rather than\nmutating the active state in place.\n\nThe affected domain owns:\n\n- the command describing the semantic change;\n- validation of its typed settings;\n- health signals that demonstrate correct operation;\n- domain-specific remediation and rollback consequences.\n\nShared policy supplies consistent evaluation, evidence, and audit mechanics.\nIdentity supplies actor attribution. Infrastructure supplies external\nconfiguration storage, service control, and other operational adapters.\n\n## Runtime change protocol\n\nLast-known-good preservation and candidate promotion are the universal runtime\nchange protocol.\n\n1. An identity proposes an immutable candidate version.\n2. Infrastructure parses it without changing active state.\n3. Each affected domain validates its typed settings.\n4. Shared policy evaluates cross-cutting constraints.\n5. Each affected capability prepares the candidate without destroying its\n active instance.\n6. Capability-specific health checks prove the candidate works.\n7. Runtime supervision atomically promotes the candidate.\n8. A bounded observation period watches for delayed failure.\n9. Failure restores the last-known-good version and records the reason,\n evidence, actor, and involved versions.\n10. Repeated remediation failure stops retrying, isolates the capability, and\n reports the fault.\n\nThe affected domain owns the semantic change command, typed validation, health\nsignals, and remediation consequences. Shared runtime supervision owns the\npromotion transaction. Infrastructure performs storage and service-control\noperations.\n\nThe protocol's exact Go contracts, storage model, atomicity boundary, and\nobservation rules require a focused runtime-supervision design pass.\n\n## Failure isolation\n\nInvalid input MUST leave the active valid configuration unchanged and produce a\ndiagnostic record.\n\nA capability that cannot start or reload SHOULD fail independently where its\nabsence does not violate a declared application invariant. Unaffected\ncapabilities continue operating.\n\nRemediation MUST be bounded, observable, and policy-driven. Recovery MUST NOT\nsilently oscillate between versions or retry forever.\n\n## Database settings: degrade paths and recovery (2026-09)\n\nThe control plane owns runtime-tunable settings and stores them in `archie.db`.\nThis replaced a dashboard overlay store that degraded to file config on any\nfailure and had three documented ways back. The replacement has none, and the\nposture is the opposite: **the daemon fails closed.**\n\n### Degrade paths\n\n| Failure | Boot behaviour | Where the operator sees it |\n| --- | --- | --- |\n| State Store unreachable | `boot.loadRuntimeConfig` returns the dial error and `archied` exits non-zero | stderr: `runtime settings unavailable` |\n| A stored resource fails validation | Same: `validate database settings` wraps the error and the daemon exits | Same |\n| SIGHUP reload with a bad file, or a control plane that cannot be read | Running config is kept; the reload records `last_error` / `last_error_at`. The reload re-applies the database layer, so it never publishes file values over the running database-owned ones | `/api/config` → `reload.last_error` → dashboard banner |\n\nFailing closed is deliberate for settings the database owns: falling back to the\nfile values would restore the second source of truth the control plane exists to\nremove. The cost is that a bad stored value stops the daemon starting, and the\nvalue can only be corrected through the daemon's own API.\n\n### Recovery: offline commands on the State Store\n\nNothing skips the control plane at boot, and the only writer for a resource is\n`ControlPlaneService.Command` on a running State Store -- which is exactly what\nthe daemon fails closed without. `archie-state-store` therefore carries the\noffline recovery commands (`archie-core-6zw0`). They work on the task database\nfile directly -- the configured `db_path` with `-tasks.sqlite` appended, not\n`db_path` itself -- and need neither `archied` nor the Web UI running: `validate`\nreads the configuration file the daemon boots with, which is what makes its\nverdict the daemon's verdict.\n\n| Command | What it does | State Store |\n| --- | --- | --- |\n| `backup -db F -out F` | writes a transactionally consistent snapshot (`VACUUM INTO`), verified as a usable store before it is published | may be running |\n| `restore -db F -from F` | replaces the database with a snapshot and drops the replaced file's `-wal`/`-shm` | must be stopped |\n| `validate -db F [-config C]` | runs the checks that stop archied starting: the file is readable, uncorrupted and no newer than the binary; every stored resource decodes under the definition that owns it; and `configuration.Validate` accepts the settings the daemon would read -- the stored ones, or the seed the State Store writes for a kind the file does not hold yet -- layered onto `C`, the check the daemon reports as `validate database settings` | may be running |\n| `rollback -db F -kind K [-revision N]` | replays an earlier revision of a stored resource through the ordinary replace, recording the rollback as a new revision | must be stopped |\n\nThe two validation layers `validate` reports are not the same check, and it\nreports both rather than merging them: the write path's own decode is stricter\nabout shape (it rejects unknown fields boot's decode ignores), while boot is\nstricter about meaning (only boot checks `dispatch.trigger`, and a positive\n`poll_interval`). Either one refusing is a store somebody has to look at.\n\nNo rollback RPC exists and none is needed: a revision carries its own value, so\nrestoring one is the same replace the dashboard and the channels already use.\nThe offline command is that operation with the store stopped -- the one state in\nwhich the daemon's own API cannot serve it.\n\n\"Must be stopped\" is checked rather than assumed. The serving process takes an\nexclusive flock on `\u003cdb\u003e.lock` for its whole life, and `restore` and `rollback`\nrefuse with `task database is owned by another process` while another process\nholds it. The read-only commands -- `backup` and `validate` -- take no lock and\nare meant to run against a serving store: `backup` because the update installer\nruns inside the process it is replacing and cannot stop the State Store, and\n`validate` because an operator checks a running deployment with it. That\n`backup` exemption is only sound while backup never replaces the database, so\nnaming the database as its own `-out` is refused. `scripts/archie-update-install`\ntakes its pre-flight snapshot through this subcommand rather than through its\nown `sqlite3` shell, so the installer and an operator run one implementation.\n\n`rollback` is the surgical route back from a bad stored value; `restore` is the\nblunt one, and is also the way back from a schema migration a release cannot\ngrow out of.\n\n## Gateway restart: two constraints learned the hard way\n\nTelegram `/restart` (`internal/channels/telegram/restart.go` plus the `Start`\nsupervisor loop). Recovered 2026-08-09 from the pre-migration issue tracker.\nBoth constraints must survive any refactor.\n\n**1. Deadlock.** The `/restart` handler runs *on* the bot instance the supervisor\nis about to stop. Tearing down inline from the handler deadlocks the restart it\njust asked for. So the handler sends on a **buffered** channel (`restartCh`,\ncapacity 1) and returns immediately; the supervisor loop in `Start` does the\nteardown. Duplicate requests hit the default branch and are dropped, not queued.\n\n**2. Lockout.** A failed config reload must **not** be fatal. It logs and\nrelaunches with the previous in-memory settings. If a bad config edit killed the\ngateway, it would also destroy the only means of fixing it remotely — which is\nthe entire reason `/restart` exists.\n\n**Scope is gateway-only by design.** The daemon keeps running so in-flight agent\ntasks survive. Exiting the process would also restart under a `unless-stopped`\npolicy, but would kill running work. `Reload` (supplied by `cmd/archied/main.go`)\nre-reads config from disk and refreshes **only** the token and `AllowedUserIDs`;\neverything else is wired into the daemon at construction and still needs a full\nrestart.\n\n**Authorisation is inherited:** `authorizedMessage` rejects non-allowlisted\nsenders before any handler runs, so all gateway commands are admin-only by\nconstruction. Do not add per-command auth without checking that invariant still\nholds.\n" + "body": "# Safe Change and Recovery\n\n**Status:** Requirements approved; mechanics are under design. Database-backed settings are implemented through the control plane (2026-09): see the degrade/recovery section below, which records a gap rather than a solution. The full runtime-supervision protocol (bounded observation, versioned audit trail) remains under design.\n**Date:** 2026-07-28 \n**Tracking issue:** [#73](https://github.com/samcharles93/archie-core/issues/73)\n\n## Purpose\n\nArchie is built and improved by agents as well as humans. The architecture MUST\nmake a change easy to locate, understand, test, audit, extend, and reverse\nwithout requiring its author to understand unrelated parts of the application.\n\nSeparation of concerns is therefore an operational safety boundary, not only a\ncode-organisation preference.\n\n## Required outcomes\n\n- An agent can improve a focused feature without receiving authority over\n unrelated domains.\n- Every event, function, action, setting, and policy has an identifiable owner\n and provenance.\n- Source and runtime changes pass deterministic checks outside the actor making\n the change.\n- A rejected or unhealthy candidate cannot replace the last known working\n version.\n- Invalid user configuration does not terminate an otherwise healthy\n application.\n- Configuration and service failures are observable, attributed, and eligible\n for bounded remediation or rollback.\n- The application continues serving unaffected capabilities when one\n capability is misconfigured or unhealthy.\n- Recovery failure is reported and contained rather than hidden in an\n unbounded retry loop.\n\n## Source changes\n\nAgent-authored source changes occur through the task-execution boundary in an\nisolated workspace. The agent does not directly mutate the running\ninstallation.\n\nPromotion is controlled by deterministic repository-owned policy, including\ntests, generated-reference drift checks, protected paths, change scope, and\nother required gates. The actor proposing a change cannot waive these checks.\n\nRepository and domain separation SHOULD allow a task to load and modify only\nthe governing requirements and code for its declared scope.\n\n## Runtime changes\n\nConfiguration and other runtime changes are proposed as candidates rather than\nmutating the active state in place.\n\nThe affected domain owns:\n\n- the command describing the semantic change;\n- validation of its typed settings;\n- health signals that demonstrate correct operation;\n- domain-specific remediation and rollback consequences.\n\nShared policy supplies consistent evaluation, evidence, and audit mechanics.\nIdentity supplies actor attribution. Infrastructure supplies external\nconfiguration storage, service control, and other operational adapters.\n\n## Runtime change protocol\n\nLast-known-good preservation and candidate promotion are the universal runtime\nchange protocol.\n\n1. An identity proposes an immutable candidate version.\n2. Infrastructure parses it without changing active state.\n3. Each affected domain validates its typed settings.\n4. Shared policy evaluates cross-cutting constraints.\n5. Each affected capability prepares the candidate without destroying its\n active instance.\n6. Capability-specific health checks prove the candidate works.\n7. Runtime supervision atomically promotes the candidate.\n8. A bounded observation period watches for delayed failure.\n9. Failure restores the last-known-good version and records the reason,\n evidence, actor, and involved versions.\n10. Repeated remediation failure stops retrying, isolates the capability, and\n reports the fault.\n\nThe affected domain owns the semantic change command, typed validation, health\nsignals, and remediation consequences. Shared runtime supervision owns the\npromotion transaction. Infrastructure performs storage and service-control\noperations.\n\nThe protocol's exact Go contracts, storage model, atomicity boundary, and\nobservation rules require a focused runtime-supervision design pass.\n\n## Failure isolation\n\nInvalid input MUST leave the active valid configuration unchanged and produce a\ndiagnostic record.\n\nA capability that cannot start or reload SHOULD fail independently where its\nabsence does not violate a declared application invariant. Unaffected\ncapabilities continue operating.\n\nRemediation MUST be bounded, observable, and policy-driven. Recovery MUST NOT\nsilently oscillate between versions or retry forever.\n\n## Database settings: degrade paths and recovery (2026-09)\n\nThe control plane owns runtime-tunable settings and stores them in `archie.db`.\nThis replaced a dashboard overlay store that degraded to file config on any\nfailure and had three documented ways back. The replacement has none, and the\nposture is the opposite: **the daemon fails closed.**\n\n### Degrade paths\n\n| Failure | Boot behaviour | Where the operator sees it |\n| --- | --- | --- |\n| State Store unreachable | `boot.loadRuntimeConfig` returns the dial error and `archied` exits non-zero | stderr: `runtime settings unavailable` |\n| A stored resource fails validation | Same: `validate database settings` wraps the error and the daemon exits | Same |\n| SIGHUP reload with a bad file, or a control plane that cannot be read | Running config is kept; the reload records `last_error` / `last_error_at`. The reload re-applies the database layer, so it never publishes file values over the running database-owned ones | `/api/config` → `reload.last_error` → dashboard banner |\n\nFailing closed is deliberate for settings the database owns: falling back to the\nfile values would restore the second source of truth the control plane exists to\nremove. The cost is that a bad stored value stops the daemon starting, and the\nvalue can only be corrected through the daemon's own API.\n\n### Recovery: offline commands on the State Store\n\nNothing skips the control plane at boot, and the only writer for a resource is\n`ControlPlaneService.Command` on a running State Store -- which is exactly what\nthe daemon fails closed without. `archie-state-store` therefore carries the\noffline recovery commands (`archie-core-6zw0`). They work on the task database\nfile directly -- the configured `db_path` with `-tasks.sqlite` appended, not\n`db_path` itself -- and need neither `archied` nor the Web UI running: `validate`\nreads the configuration file the daemon boots with, which is what makes its\nverdict the daemon's verdict.\n\n| Command | What it does | State Store |\n| --- | --- | --- |\n| `backup -db F -out F` | writes a transactionally consistent snapshot (`VACUUM INTO`), verified as a usable store before it is published | may be running |\n| `restore -db F -from F` | replaces the database with a snapshot and drops the replaced file's `-wal`/`-shm` | must be stopped |\n| `validate -db F [-config C]` | runs the checks that stop archied starting: the file is readable, uncorrupted and no newer than the binary; every stored resource decodes under the definition that owns it; and `configuration.Validate` accepts the settings the daemon would read -- the stored ones, or the seed the State Store writes for a kind the file does not hold yet -- layered onto `C`, the check the daemon reports as `validate database settings` | may be running |\n| `rollback -db F -kind K [-revision N]` | replays an earlier revision of a stored resource through the ordinary replace, recording the rollback as a new revision | must be stopped |\n\nThe two validation layers `validate` reports are not the same check, and it\nreports both rather than merging them: the write path's own decode is stricter\nabout shape (it rejects unknown fields boot's decode ignores), while boot is\nstricter about meaning (only boot checks `dispatch.trigger`, and a positive\n`poll_interval`). Either one refusing is a store somebody has to look at.\n\nNo rollback RPC exists and none is needed: a revision carries its own value, so\nrestoring one is the same replace the dashboard and the channels already use.\nThe offline command is that operation with the store stopped -- the one state in\nwhich the daemon's own API cannot serve it.\n\n\"Must be stopped\" is checked rather than assumed. The serving process takes an\nexclusive flock on `\u003cdb\u003e.lock` for its whole life, and `restore` and `rollback`\nrefuse with `task database is owned by another process` while another process\nholds it. The read-only commands -- `backup` and `validate` -- take no lock and\nare meant to run against a serving store: `backup` because the update installer\nruns inside the process it is replacing and cannot stop the State Store, and\n`validate` because an operator checks a running deployment with it. That\n`backup` exemption is only sound while backup never replaces the database, so\nnaming the database as its own `-out` is refused. `scripts/archie-update-install`\ntakes its pre-flight snapshot through this subcommand rather than through its\nown `sqlite3` shell, so the installer and an operator run one implementation.\n\n`rollback` is the surgical route back from a bad stored value; `restore` is the\nblunt one, and is also the way back from a schema migration a release cannot\ngrow out of.\n\n### The same commands on PostgreSQL\n\nWithout `-db`, each command works on the PostgreSQL database the configuration's\n`database_url` names (`internal/app/archied/state_store_recovery_postgres.go`,\n`internal/infrastructure/postgres/maintenance.go`). One database backs every\nservice, so the scope is always the whole database.\n\n- `backup -out F` runs `pg_dump --format=custom`, reads the archive back with\n `pg_restore --list` and only then replaces `F`. It takes no claim and runs\n against a serving database.\n- `restore -from F` is destructive and offline. It claims every service role's\n ownership lock (`state-store` and `gateway`) without waiting and refuses if\n any is held or the check cannot complete, then drops tables the snapshot does\n not hold and runs `pg_restore --clean --single-transaction`. Every write made\n after the snapshot is lost.\n- `validate` refuses a schema newer than the binary's highest migration, then\n runs the same two validation layers.\n- `rollback` holds the `state-store` claim for the replay.\n\nThere is no automatic schema rollback. goose `Down` drops tables, so it is data\nloss, not a rollback. `scripts/archie-update-install` refuses a daemon update\nit cannot snapshot first, and on a failed update `scripts/archie-update-watchdog`\nrolls the binaries back, leaves the database alone and prints the `restore`\ncommand for the pre-update snapshot. Whether to run it is the operator's call.\n\n## Gateway restart: two constraints learned the hard way\n\nTelegram `/restart` (`internal/channels/telegram/restart.go` plus the `Start`\nsupervisor loop). Recovered 2026-08-09 from the pre-migration issue tracker.\nBoth constraints must survive any refactor.\n\n**1. Deadlock.** The `/restart` handler runs *on* the bot instance the supervisor\nis about to stop. Tearing down inline from the handler deadlocks the restart it\njust asked for. So the handler sends on a **buffered** channel (`restartCh`,\ncapacity 1) and returns immediately; the supervisor loop in `Start` does the\nteardown. Duplicate requests hit the default branch and are dropped, not queued.\n\n**2. Lockout.** A failed config reload must **not** be fatal. It logs and\nrelaunches with the previous in-memory settings. If a bad config edit killed the\ngateway, it would also destroy the only means of fixing it remotely — which is\nthe entire reason `/restart` exists.\n\n**Scope is gateway-only by design.** The daemon keeps running so in-flight agent\ntasks survive. Exiting the process would also restart under a `unless-stopped`\npolicy, but would kill running work. `Reload` (supplied by `cmd/archied/main.go`)\nre-reads config from disk and refreshes **only** the token and `AllowedUserIDs`;\neverything else is wired into the daemon at construction and still needs a full\nrestart.\n\n**Authorisation is inherited:** `authorizedMessage` rejects non-allowlisted\nsenders before any handler runs, so all gateway commands are admin-only by\nconstruction. Do not add per-command auth without checking that invariant still\nholds.\n" }, { "url": "/dev/prds/01-project-management/", diff --git a/internal/app/archied/state_store_recovery.go b/internal/app/archied/state_store_recovery.go index fe4636726..27b81cd2a 100644 --- a/internal/app/archied/state_store_recovery.go +++ b/internal/app/archied/state_store_recovery.go @@ -5,12 +5,12 @@ // State Store is up -- is unavailable in exactly the case that needs it // (docs/architecture/safe-change-and-recovery.md). // -// Every operation works on the task database file directly, with the State -// Store stopped, and the ones that write refuse when another process owns the -// file. Backup is the exception and deliberately so: the update installer -// cannot stop the process that runs it, so the snapshot that makes a failed -// update reversible is taken against a serving store through SQLite's own -// consistent copy. +// With -db an operation works on the legacy SQLite task file; without it, on +// the PostgreSQL database database_url names (state_store_recovery_postgres.go). +// The ones that write refuse while a serving process owns the store. Backup is +// the exception and deliberately so: the update installer cannot stop the +// process that runs it, so the snapshot that makes a failed update recoverable +// is taken against a serving store through a consistent copy. // // The control-plane server both validate and rollback read through is built by // openStateStoreControlPlane -- the same root helper the served path uses -- so @@ -25,6 +25,7 @@ import ( "fmt" "time" + "github.com/samcharles93/archie-core/internal/app/controlplane" "github.com/samcharles93/archie-core/internal/config" pb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1" "github.com/samcharles93/archie-core/internal/infrastructure/configuration" @@ -76,31 +77,33 @@ type StateStoreRecoveryOptions struct { // RunStateStoreRecovery performs one offline operation on the task database and // returns the line the command reports on stdout. func RunStateStoreRecovery(ctx context.Context, options StateStoreRecoveryOptions) (string, error) { + // -db names the legacy SQLite task file; without it the command works on + // the PostgreSQL database the configuration names. + if options.DB == "" { + return runPostgresRecovery(ctx, options) + } switch options.Operation { case RecoveryBackup: - if options.DB == "" || options.Out == "" { - return "", errors.New("backup requires -db and -out") + if options.Out == "" { + return "", errors.New("backup requires -out") } if err := store.Backup(ctx, options.DB, options.Out); err != nil { return "", err } return fmt.Sprintf("backed up %s to %s", options.DB, options.Out), nil case RecoveryRestore: - if options.DB == "" || options.From == "" { - return "", errors.New("restore requires -db and -from") + if options.From == "" { + return "", errors.New("restore requires -from") } if err := store.Restore(ctx, options.DB, options.From); err != nil { return "", err } return fmt.Sprintf("restored %s from %s; start the State Store again", options.DB, options.From), nil case RecoveryValidate: - if options.DB == "" { - return "", errors.New("validate requires -db") - } return validateStore(ctx, options) case RecoveryRollback: - if options.DB == "" || options.Kind == "" { - return "", errors.New("rollback requires -db and -kind") + if options.Kind == "" { + return "", errors.New("rollback requires -kind") } return rollbackResource(ctx, options) default: @@ -146,34 +149,45 @@ func validateStore(ctx context.Context, options StateStoreRecoveryOptions) (stri // with the writer's validator, and boot's gate is the config check alone. // That is the same stance the layering takes for a kind the table is missing: // what the daemon reads for a kind nothing was ever stored for is the seed. - checked := 0 stored, err := st.StoresResources(ctx) if err != nil { return "", err } - if stored { - if checked, err = server.ValidateStored(ctx); err != nil { - return "", err - } - } - - base, err := bootConfig(ctx, options) + checked, err := checkBootGate(ctx, server, stored, options) if err != nil { return "", err } + return fmt.Sprintf("%s is a valid store at schema version %d; %d stored resources validate", options.DB, version, checked), nil +} + +// checkBootGate runs the writer's per-resource validation over what is stored, +// then boot's gate over the file config with the stored resources layered on, +// and returns how many stored resources validated. +func checkBootGate(ctx context.Context, server *controlplane.Server, stored bool, options StateStoreRecoveryOptions) (int, error) { + checked := 0 + if stored { + var err error + if checked, err = server.ValidateStored(ctx); err != nil { + return 0, err + } + } + base, err := bootConfig(ctx, options) + if err != nil { + return 0, err + } document := base if stored { document, _, err = server.StoredRuntimeConfig(ctx, base) if err != nil { - return "", err + return 0, err } } if err := configuration.Validate(&document); err != nil { // The daemon's own wording, so an operator can match this failure to the // one that stopped archied starting. - return "", fmt.Errorf("validate database settings: %w", err) + return 0, fmt.Errorf("validate database settings: %w", err) } - return fmt.Sprintf("%s is a valid store at schema version %d; %d stored resources validate", options.DB, version, checked), nil + return checked, nil } // bootConfig resolves the configuration the daemon would boot with. validate is @@ -217,22 +231,27 @@ func rollbackResource(ctx context.Context, options StateStoreRecoveryOptions) (s return "", err } defer func() { _ = st.Close() }() + return replayRevision(ctx, st, options) +} - server, err := openStateStoreControlPlane(st) +// replayRevision puts back the revision options names through the control +// plane's ordinary replace. The caller holds the store's ownership claim. +func replayRevision(ctx context.Context, resources controlplane.ResourceStore, options StateStoreRecoveryOptions) (string, error) { + server, err := openStateStoreControlPlane(resources) if err != nil { return "", err } if !server.Owns(options.Kind) { return "", fmt.Errorf("unknown resource kind %q", options.Kind) } - current, err := st.Resource(ctx, options.Kind) + current, err := resources.Resource(ctx, options.Kind) if errors.Is(err, store.ErrResourceNotFound) { return "", fmt.Errorf("%s has no stored value to roll back", options.Kind) } if err != nil { return "", fmt.Errorf("read %s: %w", options.Kind, err) } - revision, err := revisionToReplay(ctx, st, options.Kind, current.Version, options.Revision) + revision, err := revisionToReplay(ctx, resources, options.Kind, current.Version, options.Revision) if err != nil { return "", err } @@ -254,7 +273,7 @@ func rollbackResource(ctx context.Context, options StateStoreRecoveryOptions) (s // revisionToReplay picks the value to put back: the revision the operator // named, or the newest one older than the value the resource carries now. -func revisionToReplay(ctx context.Context, st *store.Store, kind string, current, requested int64) (store.Resource, error) { +func revisionToReplay(ctx context.Context, st controlplane.ResourceStore, kind string, current, requested int64) (store.Resource, error) { history, err := st.ResourceHistory(ctx, kind, 0) if err != nil { return store.Resource{}, fmt.Errorf("read %s history: %w", kind, err) diff --git a/internal/app/archied/state_store_recovery_postgres.go b/internal/app/archied/state_store_recovery_postgres.go new file mode 100644 index 000000000..ace209b07 --- /dev/null +++ b/internal/app/archied/state_store_recovery_postgres.go @@ -0,0 +1,108 @@ +package archied + +import ( + "context" + "errors" + "fmt" + + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/samcharles93/archie-core/internal/infrastructure/postgres" +) + +// runPostgresRecovery performs one recovery operation on the PostgreSQL +// database the configuration names. There is no automatic schema rollback: +// goose Down drops tables, so the way back from a migration is restoring the +// snapshot taken before it, which loses every write made after that snapshot. +func runPostgresRecovery(ctx context.Context, options StateStoreRecoveryOptions) (string, error) { + cfg, err := bootConfig(ctx, options) + if err != nil { + return "", err + } + url := cfg.DatabaseURL + if url == "" { + return "", errors.New("database_url is required: the configuration names no PostgreSQL database (pass -db for a legacy SQLite task file)") + } + switch options.Operation { + case RecoveryBackup: + if options.Out == "" { + return "", errors.New("backup requires -out") + } + if err := postgres.Backup(ctx, url, options.Out); err != nil { + return "", err + } + return "backed up the whole database to " + options.Out, nil + case RecoveryRestore: + if options.From == "" { + return "", errors.New("restore requires -from") + } + if err := postgres.Restore(ctx, url, options.From); err != nil { + return "", err + } + return fmt.Sprintf("restored the whole database from %s; every write made after the snapshot is gone; start the archie services again", options.From), nil + case RecoveryValidate: + return validatePostgres(ctx, url, options) + case RecoveryRollback: + if options.Kind == "" { + return "", errors.New("rollback requires -kind") + } + return rollbackPostgres(ctx, url, options) + default: + return "", fmt.Errorf("unknown recovery command %q", options.Operation) + } +} + +// openRecoveryPool opens the database without migrating it: recovery inspects +// or repairs the schema it finds, and refuses one newer than this binary. +func openRecoveryPool(ctx context.Context, url string) (*pgxpool.Pool, int64, error) { + pool, err := postgres.Open(ctx, url) + if err != nil { + return nil, 0, err + } + version, err := postgres.SchemaVersion(ctx, pool) + if err == nil { + err = postgres.CheckSchemaVersion(version) + } + if err != nil { + pool.Close() + return nil, 0, err + } + return pool, version, nil +} + +// validatePostgres is validateStore for the PostgreSQL database. An +// unmigrated database holds nothing to check; the State Store migrates and +// seeds it on its next start, so only boot's config gate applies. +func validatePostgres(ctx context.Context, url string, options StateStoreRecoveryOptions) (string, error) { + pool, version, err := openRecoveryPool(ctx, url) + if err != nil { + return "", err + } + defer pool.Close() + server, err := openStateStoreControlPlane(postgres.NewResources(pool)) + if err != nil { + return "", err + } + checked, err := checkBootGate(ctx, server, version > 0, options) + if err != nil { + return "", err + } + return fmt.Sprintf("the database is valid at schema version %d; %d stored resources validate", version, checked), nil +} + +// rollbackPostgres replays an earlier revision holding the State Store's +// ownership claim, so it refuses while a State Store serves and none can +// start until it finishes. +func rollbackPostgres(ctx context.Context, url string, options StateStoreRecoveryOptions) (string, error) { + pool, _, err := openRecoveryPool(ctx, url) + if err != nil { + return "", err + } + defer pool.Close() + claim, err := postgres.AcquireOwnership(ctx, pool, postgres.OwnerStateStore) + if err != nil { + return "", fmt.Errorf("rollback refused, stop the State Store first: %w", err) + } + defer func() { _ = claim.Release(context.WithoutCancel(ctx)) }() + return replayRevision(ctx, postgres.NewResources(pool), options) +} diff --git a/internal/app/archied/state_store_recovery_postgres_test.go b/internal/app/archied/state_store_recovery_postgres_test.go new file mode 100644 index 000000000..57a7d482a --- /dev/null +++ b/internal/app/archied/state_store_recovery_postgres_test.go @@ -0,0 +1,201 @@ +package archied + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/samcharles93/archie-core/internal/app/controlplane" + "github.com/samcharles93/archie-core/internal/domain/storecontract" + "github.com/samcharles93/archie-core/internal/infrastructure/postgres" + "github.com/samcharles93/archie-core/internal/infrastructure/postgres/pgtest" +) + +// pgRecoveryFixture is a migrated database and the config file naming it, +// which is all the Postgres recovery path is given: it resolves database_url +// from the configuration exactly as the State Store does. +func pgRecoveryFixture(t *testing.T) (configPath string, pool *pgxpool.Pool, url string) { + t.Helper() + url = pgtest.URL(t) + dir := t.TempDir() + configPath = filepath.Join(dir, "config.toml") + body := fmt.Sprintf("bot_user = 'archie'\ndb_path = %q\ndatabase_url = %q\n[forge]\ntype = 'github'\nhost = 'https://github.example.com'\n", + filepath.Join(dir, "archie.db"), url) + if err := os.WriteFile(configPath, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + pool, err := postgres.Open(t.Context(), url) + if err != nil { + t.Fatal(err) + } + t.Cleanup(pool.Close) + if err := postgres.Migrate(t.Context(), pool, postgres.Migrations()); err != nil { + t.Fatal(err) + } + return configPath, pool, url +} + +func requirePGTools(t *testing.T) { + t.Helper() + for _, tool := range []string{"pg_dump", "pg_restore"} { + if _, err := exec.LookPath(tool); err != nil { + t.Skipf("%s is not on PATH; this test needs the PostgreSQL 18 client tools", tool) + } + } +} + +func putResource(t *testing.T, pool *pgxpool.Pool, kind, value, label string) { + t.Helper() + resources := postgres.NewResources(pool) + expected := int64(0) + if current, err := resources.Resource(t.Context(), kind); err == nil { + expected = current.Version + } + if _, err := resources.PutResource(t.Context(), storecontract.ResourceWrite{ + Kind: kind, Value: []byte(value), Actor: label, Source: "test", + RequestID: fmt.Sprintf("%s-%d", label, expected+1), ExpectedVersion: expected, + }); err != nil { + t.Fatalf("put %s: %v", kind, err) + } +} + +func holdRole(t *testing.T, url, role string) { + t.Helper() + pool, err := postgres.Open(t.Context(), url) + if err != nil { + t.Fatal(err) + } + t.Cleanup(pool.Close) + claim, err := postgres.AcquireOwnership(t.Context(), pool, role) + if err != nil { + t.Fatalf("hold %s: %v", role, err) + } + t.Cleanup(func() { _ = claim.Release(t.Context()) }) +} + +func TestPostgresRecoveryRequiresADatabaseURL(t *testing.T) { + dir := t.TempDir() + configPath := filepath.Join(dir, "config.toml") + if err := os.WriteFile(configPath, []byte("bot_user = 'archie'\n[forge]\ntype = 'github'\nhost = 'https://github.example.com'\n"), 0o600); err != nil { + t.Fatal(err) + } + for _, op := range []string{RecoveryBackup, RecoveryRestore, RecoveryValidate, RecoveryRollback} { + t.Run(op, func(t *testing.T) { + _, err := RunStateStoreRecovery(t.Context(), StateStoreRecoveryOptions{ + Operation: op, Config: configPath, Out: filepath.Join(dir, "o"), From: filepath.Join(dir, "f"), Kind: "k", + }) + if err == nil || !strings.Contains(err.Error(), "database_url") { + t.Fatalf("%s without database_url = %v, want a refusal naming database_url", op, err) + } + }) + } +} + +func TestPostgresRecoveryBackupAndRestore(t *testing.T) { + requirePGTools(t) + configPath, pool, url := pgRecoveryFixture(t) + putResource(t, pool, controlplane.ModelRoleAssignmentsKind, `{"implement":"anthropic/claude"}`, "snapshot") + snapshot := filepath.Join(t.TempDir(), "snapshot.dump") + + if _, err := RunStateStoreRecovery(t.Context(), StateStoreRecoveryOptions{Operation: RecoveryBackup, Config: configPath, Out: snapshot}); err != nil { + t.Fatalf("backup: %v", err) + } + putResource(t, pool, controlplane.ModelRoleAssignmentsKind, `{"implement":"openai/gpt"}`, "later") + + for _, role := range postgres.ServiceOwners { + t.Run("refused while "+role+" serves", func(t *testing.T) { + holdRole(t, url, role) + _, err := RunStateStoreRecovery(t.Context(), StateStoreRecoveryOptions{Operation: RecoveryRestore, Config: configPath, From: snapshot}) + if !errors.Is(err, postgres.ErrOwned) { + t.Fatalf("restore with %s serving = %v, want ErrOwned", role, err) + } + }) + } + + summary, err := RunStateStoreRecovery(t.Context(), StateStoreRecoveryOptions{Operation: RecoveryRestore, Config: configPath, From: snapshot}) + if err != nil { + t.Fatalf("restore: %v", err) + } + if !strings.Contains(summary, "after the snapshot") { + t.Errorf("restore must say writes after the snapshot are lost; summary = %q", summary) + } + current, err := postgres.NewResources(pool).Resource(t.Context(), controlplane.ModelRoleAssignmentsKind) + if err != nil { + t.Fatal(err) + } + if current.Version != 1 { + t.Errorf("restored version = %d, want the snapshot's 1", current.Version) + } +} + +func TestPostgresRecoveryValidate(t *testing.T) { + tests := []struct { + name string + setup func(t *testing.T, pool *pgxpool.Pool) + wantErr string + }{ + {name: "healthy store", setup: func(*testing.T, *pgxpool.Pool) {}}, + {name: "schema newer than this binary", wantErr: "newer than this binary", setup: func(t *testing.T, pool *pgxpool.Pool) { + if _, err := pool.Exec(t.Context(), "INSERT INTO goose_db_version (version_id, is_applied) VALUES ($1, true)", postgres.SupportedSchemaVersion()+1); err != nil { + t.Fatal(err) + } + }}, + {name: "stored value boot refuses", wantErr: "dispatch.trigger", setup: func(t *testing.T, pool *pgxpool.Pool) { + putResource(t, pool, "scheduling-policy", `{"poll_interval":"1m","max_retries":3,"dispatch":{"trigger":"bogus"}}`, "bad") + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + configPath, pool, _ := pgRecoveryFixture(t) + tt.setup(t, pool) + summary, err := RunStateStoreRecovery(t.Context(), StateStoreRecoveryOptions{Operation: RecoveryValidate, Config: configPath}) + if tt.wantErr == "" { + if err != nil { + t.Fatalf("validate: %v", err) + } + if !strings.Contains(summary, fmt.Sprintf("schema version %d", postgres.SupportedSchemaVersion())) { + t.Errorf("summary = %q, want the schema version", summary) + } + return + } + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("validate = %v, want refusal containing %q", err, tt.wantErr) + } + }) + } +} + +func TestPostgresRecoveryRollback(t *testing.T) { + configPath, pool, url := pgRecoveryFixture(t) + kind := controlplane.ModelRoleAssignmentsKind + putResource(t, pool, kind, `{"implement":"anthropic/claude"}`, "first") + putResource(t, pool, kind, `{"implement":"openai/gpt"}`, "second") + options := StateStoreRecoveryOptions{Operation: RecoveryRollback, Config: configPath, Kind: kind} + + t.Run("refused while the State Store serves", func(t *testing.T) { + holdRole(t, url, postgres.OwnerStateStore) + if _, err := RunStateStoreRecovery(t.Context(), options); !errors.Is(err, postgres.ErrOwned) { + t.Fatalf("rollback with the State Store serving = %v, want ErrOwned", err) + } + }) + + if _, err := RunStateStoreRecovery(t.Context(), options); err != nil { + t.Fatalf("rollback: %v", err) + } + current, err := postgres.NewResources(pool).Resource(t.Context(), kind) + if err != nil { + t.Fatal(err) + } + if string(current.Value) != `{"implement": "anthropic/claude"}` && string(current.Value) != `{"implement":"anthropic/claude"}` { + t.Errorf("value = %s, want the first revision's", current.Value) + } + if current.Version != 3 { + t.Errorf("version = %d, want the rollback recorded as revision 3", current.Version) + } +} diff --git a/internal/releaseupdate/db_snapshot_script_test.go b/internal/releaseupdate/db_snapshot_script_test.go new file mode 100644 index 000000000..882037a26 --- /dev/null +++ b/internal/releaseupdate/db_snapshot_script_test.go @@ -0,0 +1,114 @@ +package releaseupdate + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +const snapshotConfig = "database_url = 'postgres://archie:hunter2@db.example/archie'\n\n[containers]\nimage = 'registry.example/archie-agent:stable'\n" + +// The new State Store migrates on start and nothing undoes a migration +// automatically, so a daemon update snapshots the whole database first, +// through the release's own recovery subcommand, and hands the snapshot's +// path to the watchdog. The connection string stays inside the config. +func TestUpdateInstallSnapshotsTheDatabaseBeforeMigrating(t *testing.T) { + snapshot := filepath.Join(t.TempDir(), "pre-update.dump") + _, calls, _ := runUpdateInstallWithConfig(t, map[string]string{ + "ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0", + "ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0", + "ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0", + "ARCHIE_DB_SNAPSHOT": snapshot, + }, snapshotConfig, false) + + assertCallContains(t, calls, "go run ./cmd/archie-state-store backup", "-config ", "-out "+snapshot) + assertCallContains(t, calls, "systemd-run", "--setenv=ARCHIE_DB_SNAPSHOT="+snapshot) + assertCallAbsent(t, calls, "hunter2") +} + +func TestUpdateInstallSnapshotScope(t *testing.T) { + tests := []struct { + name string + env map[string]string + wantSnapshot bool + wantRefusal bool + }{ + {name: "agent-only update does not touch the database", env: map[string]string{ + "ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0", "ARCHIE_UPDATE_AGENT_VERSION": "1.22.0", + }}, + {name: "daemon update refuses when the snapshot fails", wantSnapshot: true, wantRefusal: true, env: map[string]string{ + "ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0", "ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0", + "ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0", "ARCHIE_TEST_GO_FAILS_SNAPSHOT": "1", + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, calls, output := runUpdateInstallWithConfig(t, tt.env, snapshotConfig, tt.wantRefusal) + snapshotted := false + for _, call := range calls { + if strings.Contains(call, "archie-state-store backup") && strings.Contains(call, "-config ") { + snapshotted = true + } + } + if snapshotted != tt.wantSnapshot { + t.Fatalf("snapshot attempted = %v, want %v; calls = %q", snapshotted, tt.wantSnapshot, calls) + } + if !tt.wantRefusal { + return + } + if !strings.Contains(output, "could not snapshot the database") { + t.Errorf("refusal must say the snapshot failed; output = %q", output) + } + for _, call := range calls { + if strings.Contains(call, "install -m755") && !strings.Contains(call, "/scripts/") { + t.Errorf("refused update still installed %q", call) + } + } + }) + } +} + +// A failed daemon update rolls the binaries back but never the database, and +// says so, naming the explicit offline restore rather than running one. +func TestUpdateWatchdogNeverRestoresTheDatabaseItself(t *testing.T) { + root, err := filepath.Abs(filepath.Join("..", "..")) + if err != nil { + t.Fatal(err) + } + work := t.TempDir() + binDir, fakeDir := filepath.Join(work, "bin"), filepath.Join(work, "fake-bin") + if err := os.MkdirAll(binDir, 0o755); err != nil { + t.Fatal(err) + } + for name, content := range map[string]string{"archied": "new", "archied.prev": "old"} { + if err := os.WriteFile(filepath.Join(binDir, name), []byte(content), 0o755); err != nil { + t.Fatal(err) + } + } + callsPath := filepath.Join(work, "calls") + writeFakeCommand(t, fakeDir, "systemctl", `exit 0`) + writeFakeCommand(t, fakeDir, "curl", `exit 1`) + for _, tool := range []string{"pg_restore", "psql", "go", "archie-state-store"} { + writeFakeCommand(t, fakeDir, tool, `printf '%s\n' "`+tool+` $*" >> "$ARCHIE_TEST_CALLS"`) + } + snapshot := filepath.Join(work, "pre-update.dump") + cmd := exec.CommandContext(t.Context(), filepath.Join(root, "scripts", "archie-update-watchdog")) + cmd.Env = append(os.Environ(), "PATH="+fakeDir+":"+os.Getenv("PATH"), "ARCHIE_BIN_DIR="+binDir, + "ARCHIE_TEST_CALLS="+callsPath, "ARCHIE_UPDATE_REPORT_PATH="+filepath.Join(work, "report.json"), + "ARCHIE_UPDATE_HEALTH_TIMEOUT=0", "ARCHIE_UPDATE_COMPONENTS=daemon", + "ARCHIE_DB_SNAPSHOT="+snapshot, "ARCHIE_CONFIG_PATH=/etc/archie/config.toml") + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("watchdog failed: %v\n%s", err, output) + } + if calls, _ := os.ReadFile(callsPath); len(calls) != 0 { + t.Errorf("watchdog ran database tooling itself: %q", calls) + } + for _, want := range []string{"NOT rolled back", "archie-state-store restore -config /etc/archie/config.toml -from " + snapshot, "lost"} { + if !strings.Contains(string(output), want) { + t.Errorf("output lacks %q:\n%s", want, output) + } + } +} diff --git a/internal/releaseupdate/install_script_test.go b/internal/releaseupdate/install_script_test.go index 9b8737af1..021632866 100644 --- a/internal/releaseupdate/install_script_test.go +++ b/internal/releaseupdate/install_script_test.go @@ -411,6 +411,8 @@ printf '%s\n' "go $*" >> "$ARCHIE_TEST_CALLS" # must refuse the update. if [ "$1" = "run" ]; then [ "${ARCHIE_TEST_GO_FAILS_RUN:-}" = "1" ] && exit 1 + # The PostgreSQL snapshot is the backup run that names no -db file. + [ "${ARCHIE_TEST_GO_FAILS_SNAPSHOT:-}" = "1" ] && [[ " $* " != *" -db "* ]] && exit 1 exit 0 fi out="" diff --git a/scripts/archie-update-install b/scripts/archie-update-install index f108d0d1d..f19aec273 100755 --- a/scripts/archie-update-install +++ b/scripts/archie-update-install @@ -80,6 +80,7 @@ AGENT_VERSION_FILE="${ARCHIE_AGENT_VERSION_FILE:-$BIN_DIR/archie-agent.version}" AGENT_VERSION_BACKUP="${ARCHIE_AGENT_VERSION_BACKUP:-$BIN_DIR/archie-agent.version.prev}" TASK_DB_PATH="${ARCHIE_TASK_DB_PATH:-$(configured_task_db_path)}" TASK_DB_BACKUP="${ARCHIE_TASK_DB_BACKUP:-$BIN_DIR/archie-task-db.prev.sqlite}" +DB_SNAPSHOT="${ARCHIE_DB_SNAPSHOT:-$BIN_DIR/archie-db.pre-update.dump}" # The address the daemon dials for its state, so the watchdog can wait for it to # accept connections before it starts archied. Read with the same narrow awk as # the task database path and for the same reason: the daemon stays the authority @@ -284,6 +285,22 @@ if [[ ",$components," == *,daemon,* ]] && [ -n "$TASK_DB_PATH" ] && [ -f "$TASK_ fi fi +# The pre-migration snapshot of the PostgreSQL database: the whole database, +# since every service shares it. The new State Store migrates on start and a +# migration is never undone automatically (goose Down drops tables), so this +# snapshot is the only way back, restored by an operator with every service +# stopped. The recovery subcommand resolves database_url from the same config +# the services boot with, so the connection string never passes through this +# script. No snapshot, no update. +if [[ ",$components," == *,daemon,* ]]; then + echo "==> snapshotting the database before the update migrates it" + mkdir -p "$(dirname "$DB_SNAPSHOT")" + if ! go run ./cmd/archie-state-store backup -config "$CONFIG_PATH" -out "$DB_SNAPSHOT"; then + echo "refusing to install $GW: could not snapshot the database named by $CONFIG_PATH, so a failed migration could not be recovered" >&2 + exit 1 + fi +fi + # Refresh adapters from the approved release source. The running shell # continues with this copy; subsequent updates use the current contract. install -m755 "$source_dir/scripts/archie-update-install" "$BIN_DIR/archie-update-install" @@ -377,6 +394,8 @@ systemd-run --user --unit=archied-update-restart --collect --quiet \ --setenv=ARCHIE_AGENT_VERSION_PREVIOUS_EXISTS="${AGENT_VERSION_PREVIOUS_EXISTS:-0}" \ --setenv=ARCHIE_TASK_DB_PATH="$TASK_DB_PATH" \ --setenv=ARCHIE_TASK_DB_BACKUP="$TASK_DB_BACKUP" \ + --setenv=ARCHIE_DB_SNAPSHOT="$DB_SNAPSHOT" \ + --setenv=ARCHIE_CONFIG_PATH="$CONFIG_PATH" \ --setenv=ARCHIE_UPDATE_STATE_STORE_TARGET="$STATE_STORE_TARGET" \ "$BIN_DIR/archie-update-watchdog" diff --git a/scripts/archie-update-watchdog b/scripts/archie-update-watchdog index b657833e1..a867767b3 100755 --- a/scripts/archie-update-watchdog +++ b/scripts/archie-update-watchdog @@ -15,6 +15,8 @@ AGENT_VERSION_BACKUP="${ARCHIE_AGENT_VERSION_BACKUP:-$BIN_DIR/archie-agent.versi AGENT_VERSION_PREVIOUS_EXISTS="${ARCHIE_AGENT_VERSION_PREVIOUS_EXISTS:-0}" TASK_DB_PATH="${ARCHIE_TASK_DB_PATH:-}" TASK_DB_BACKUP="${ARCHIE_TASK_DB_BACKUP:-$BIN_DIR/archie-task-db.prev.sqlite}" +DB_SNAPSHOT="${ARCHIE_DB_SNAPSHOT:-$BIN_DIR/archie-db.pre-update.dump}" +CONFIG_PATH="${ARCHIE_CONFIG_PATH:-${XDG_CONFIG_HOME:-$HOME/.config}/archie/config.toml}" # The daemon exports its own health address (releaseupdate.CommandInstaller); # this default only covers a hand-driven run, and must stay the address archied # itself serves /healthz on -- [health] listen, defaulted in @@ -218,6 +220,16 @@ rollback_task_db() { rm -f "$TASK_DB_PATH-wal" "$TASK_DB_PATH-shm" } +# explain_database_recovery states what a daemon rollback does not do. The +# PostgreSQL database is never restored automatically: goose Down drops tables, +# and a restore is destructive, loses every write after the snapshot and needs +# every service stopped, so it is an operator's decision, not this script's. +explain_database_recovery() { + echo "==> the PostgreSQL database was NOT rolled back; schema migrations are never undone automatically" >&2 + echo "==> if the previous release refuses the migrated schema, stop every archie service and restore the pre-update snapshot (every write made after it is lost):" >&2 + echo "==> archie-state-store restore -config $CONFIG_PATH -from $DB_SNAPSHOT" >&2 +} + start_units if wait_healthy; then @@ -273,5 +285,8 @@ fi if [ "$rolled_back" = true ]; then start_units fi +if [[ ",$COMPONENTS," == *,daemon,* ]]; then + explain_database_recovery +fi write_report "failed" "$rolled_back"