From 014868eda8c0fd9d64cfc634a9fdb197bcd6ca2d Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Mon, 21 Sep 2026 20:20:40 +1000 Subject: [PATCH 1/4] feat(statestore): offline backup, restore, validate and rollback archie-state-store gains the offline recovery commands docs/prds/runtime-control-plane.md ("Bootstrap, migration, and recovery") asks for. They work on the task database file directly -- -tasks.sqlite, not db_path -- and need neither archied nor the Web UI, which matters because the failure they exist for is the one that stops archied starting: backup -db F -out F consistent snapshot (VACUUM INTO), verified as a usable store before it is published; safe against a serving store restore -db F -from F replace the database with a snapshot, dropping the replaced file's -wal/-shm validate -db F readable, not corrupt, schema no newer than this binary, every stored resource decoding under the definition that owns it rollback -db F -kind K replay an earlier revision of a stored resource [-revision N] through the ordinary replace, recorded as a new revision No rollback RPC is added and none is needed: a revision carries its own value, so restoring one is the same replace the dashboard and the channels already use, by hand, with the store stopped. "The State Store is stopped" is checked rather than assumed. The serving process now takes an exclusive flock on .lock for its whole life (store.AcquireOwnership), and restore/rollback refuse with `task database is owned by another process` while it holds it. backup is the deliberate exception: the update installer runs inside the process it is replacing and cannot stop the State Store. scripts/archie-update-install takes its pre-flight snapshot through `go run ./cmd/archie-state-store backup` from the release being installed instead of its own sqlite3 shell, so the installer and an operator run one implementation. The watchdog's refusal semantics are unchanged: an update that cannot back up still must not start. Docs: safe-change-and-recovery.md's "Recovery: not yet built" now describes what exists. --- cmd/archie-state-store/main.go | 12 + cmd/archie-state-store/recovery.go | 74 +++ cmd/archie-state-store/recovery_test.go | 544 ++++++++++++++++++ docs/architecture/safe-change-and-recovery.md | 43 +- internal/app/archied/state_store.go | 20 +- internal/app/archied/state_store_recovery.go | 210 +++++++ internal/app/controlplane/registry.go | 36 ++ internal/releaseupdate/install_script_test.go | 20 +- internal/store/maintenance.go | 253 ++++++++ internal/store/maintenance_test.go | 92 +++ internal/store/ownership.go | 68 +++ internal/store/ownership_unix.go | 30 + internal/store/ownership_windows.go | 23 + scripts/archie-update-install | 10 +- 14 files changed, 1417 insertions(+), 18 deletions(-) create mode 100644 cmd/archie-state-store/recovery.go create mode 100644 cmd/archie-state-store/recovery_test.go create mode 100644 internal/app/archied/state_store_recovery.go create mode 100644 internal/store/maintenance.go create mode 100644 internal/store/maintenance_test.go create mode 100644 internal/store/ownership.go create mode 100644 internal/store/ownership_unix.go create mode 100644 internal/store/ownership_windows.go diff --git a/cmd/archie-state-store/main.go b/cmd/archie-state-store/main.go index 130956aad..6ca5ba8a5 100644 --- a/cmd/archie-state-store/main.go +++ b/cmd/archie-state-store/main.go @@ -15,6 +15,7 @@ import ( "os" "os/signal" "path/filepath" + "strings" "syscall" "github.com/samcharles93/archie-core/internal/app/archied" @@ -23,6 +24,13 @@ import ( func main() { os.Exit(run()) } func run() int { + // A leading non-flag argument selects a recovery subcommand; anything else, + // including no arguments at all, serves the State Store contract. An + // unrecognised word is refused rather than ignored, so a mistyped recovery + // command never leaves an operator watching a server start instead. + if args := os.Args[1:]; len(args) > 0 && !strings.HasPrefix(args[0], "-") { + return runRecovery(args, os.Stdout, os.Stderr) + } base, err := os.UserConfigDir() if err != nil { fmt.Fprintln(os.Stderr, err) @@ -34,6 +42,10 @@ func run() int { flag.StringVar(&options.Listen, "listen", "", "state store gRPC listen address (defaults to [services.state].listen, else 127.0.0.1:9090)") flag.StringVar(&options.Token, "token", "", "bearer token required for a non-loopback listener (defaults to [services.state].target_token)") flag.StringVar(&options.ReadyAddr, "ready-addr", "", "optional readiness HTTP listen address (e.g. 127.0.0.1:9091)") + flag.Usage = func() { + fmt.Fprintf(flag.CommandLine.Output(), "usage: archie-state-store [flags] # serve the State Store gRPC contract\n\noffline recovery: archie-state-store -h\n\n") + flag.PrintDefaults() + } flag.Parse() ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() diff --git a/cmd/archie-state-store/recovery.go b/cmd/archie-state-store/recovery.go new file mode 100644 index 000000000..00ee75750 --- /dev/null +++ b/cmd/archie-state-store/recovery.go @@ -0,0 +1,74 @@ +package main + +import ( + "context" + "flag" + "fmt" + "io" + + "github.com/samcharles93/archie-core/internal/app/archied" +) + +// recoveryUsage is the whole flag surface of the offline recovery commands. +// They are subcommands of this binary rather than a recovery binary of their +// own: offline database maintenance belongs to the process that owns the file +// and its path layout (docs/architecture/organisation.md#process-boundaries). +const recoveryUsage = `usage: archie-state-store [flags] + +With no command, archie-state-store serves the State Store gRPC contract. + +The recovery commands operate on the task database file directly -- the +configured db_path with "-tasks.sqlite" appended -- and are how an operator +recovers a database the control plane will not start on. Stop the State Store +first: restore and rollback rewrite the file and refuse while another process +owns it, while backup takes a consistent snapshot of a serving store. + + backup -db FILE -out FILE write a snapshot of the database + restore -db FILE -from FILE replace the database with a snapshot + validate -db FILE check the database the way boot does + rollback -db FILE -kind KIND replay an earlier revision of a stored + [-revision N] resource through the ordinary replace +` + +// runRecovery parses and performs one offline recovery command, reporting the +// operation's summary on stdout. Exit codes follow the serve path: 0 succeeded, +// 1 the operation failed, 2 the command line was wrong. +func runRecovery(args []string, stdout, stderr io.Writer) int { + if len(args) == 0 { + fmt.Fprint(stderr, recoveryUsage) + return 2 + } + command := args[0] + options := archied.StateStoreRecoveryOptions{Operation: command} + switch command { + case archied.RecoveryBackup, archied.RecoveryRestore, archied.RecoveryValidate, archied.RecoveryRollback: + default: + fmt.Fprintf(stderr, "archie-state-store: unknown command %q\n\n", command) + fmt.Fprint(stderr, recoveryUsage) + return 2 + } + + flags := flag.NewFlagSet("archie-state-store "+command, flag.ContinueOnError) + flags.SetOutput(stderr) + flags.Usage = func() { fmt.Fprint(stderr, recoveryUsage) } + flags.StringVar(&options.DB, "db", "", "task database file the State Store owns (-tasks.sqlite)") + flags.StringVar(&options.Out, "out", "", "snapshot file backup writes") + flags.StringVar(&options.From, "from", "", "snapshot file restore reads") + flags.StringVar(&options.Kind, "kind", "", "control-plane resource kind rollback replays") + flags.Int64Var(&options.Revision, "revision", 0, "revision rollback replays (default: the newest one older than the current value)") + if err := flags.Parse(args[1:]); err != nil { + return 2 + } + if flags.NArg() > 0 { + fmt.Fprintf(stderr, "archie-state-store %s: unexpected argument %q\n", command, flags.Arg(0)) + return 2 + } + + summary, err := archied.RunStateStoreRecovery(context.Background(), options) + if err != nil { + fmt.Fprintf(stderr, "archie-state-store %s: %v\n", command, err) + return 1 + } + fmt.Fprintln(stdout, summary) + return 0 +} diff --git a/cmd/archie-state-store/recovery_test.go b/cmd/archie-state-store/recovery_test.go new file mode 100644 index 000000000..efe96c4f9 --- /dev/null +++ b/cmd/archie-state-store/recovery_test.go @@ -0,0 +1,544 @@ +// Package main tests the offline recovery subcommands of archie-state-store: +// backup, restore, validate and rollback against the task database file +// directly. They are the operator's only path back when the control plane's +// stored settings will not validate, because the daemon fails closed: the +// in-band remedy (replay an earlier revision while the State Store is up) is +// unavailable in exactly the case that needs it. +// +// The tests drive the subcommands the way an operator does -- through +// runRecovery, against real files on disk -- rather than through the internal +// helpers, so the flag surface and the exit codes are part of what is verified. + +package main + +import ( + "bytes" + "database/sql" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + _ "modernc.org/sqlite" + + "github.com/samcharles93/archie-core/internal/app/archied" + "github.com/samcharles93/archie-core/internal/app/controlplane" + "github.com/samcharles93/archie-core/internal/store" +) + +// runRecoveryCmd runs one subcommand the way main does and returns its exit +// code with both streams captured. +func runRecoveryCmd(t *testing.T, args ...string) (int, string, string) { + t.Helper() + var stdout, stderr bytes.Buffer + code := runRecovery(args, &stdout, &stderr) + return code, stdout.String(), stderr.String() +} + +// openTaskStore opens the task database the subcommands operate on. It does not +// take the ownership lock: only the serving process does that, and a test that +// wants a "live" store holds it explicitly with holdStoreOwnership. +func openTaskStore(t *testing.T, path string) *store.Store { + t.Helper() + st, err := store.Open(t.Context(), path) + if err != nil { + t.Fatalf("open task store %s: %v", path, err) + } + t.Cleanup(func() { _ = st.Close() }) + return st +} + +// holdStoreOwnership takes the ownership lock the running State Store holds for +// its whole life, so a test can prove a command refuses to touch a live store. +func holdStoreOwnership(t *testing.T, path string) { + t.Helper() + ownership, err := store.AcquireOwnership(path) + if err != nil { + t.Fatalf("acquire store ownership: %v", err) + } + t.Cleanup(func() { _ = ownership.Release() }) +} + +// seedTask writes one task so a snapshot can be distinguished from the live +// database by what it contains, not merely by its existence. +func seedTask(t *testing.T, st *store.Store, title string) { + t.Helper() + if _, err := st.EnqueueChatTask(t.Context(), "acme", "widget", title, "body", "implement", ""); err != nil { + t.Fatalf("seed task %q: %v", title, err) + } +} + +func taskTitles(t *testing.T, path string) []string { + t.Helper() + st := openTaskStore(t, path) + tasks, err := st.Tasks(t.Context(), 100) + if err != nil { + t.Fatalf("list tasks in %s: %v", path, err) + } + titles := make([]string, 0, len(tasks)) + for _, task := range tasks { + titles = append(titles, task.Title) + } + return titles +} + +// backupPath is the snapshot every restore test restores from. +func backupPath(t *testing.T, dir string) string { return filepath.Join(dir, "snapshot.sqlite") } + +// backup takes the snapshot the caller then restores, through the subcommand +// under test rather than through the store directly, so a broken backup fails +// the restore test too instead of silently passing an empty file. +func backup(t *testing.T, db, out string) { + t.Helper() + code, _, stderr := runRecoveryCmd(t, "backup", "-db", db, "-out", out) + if code != 0 { + t.Fatalf("backup exited %d: %s", code, stderr) + } +} + +// The update installer cannot stop the process that runs it, so it backs the +// task database up while the State Store is serving. VACUUM INTO is safe +// against a live writer; a backup that insisted on an exclusively-owned file +// would break the one caller it has, and the update would refuse to start. +func TestRecoveryBackupSnapshotsAStoreItsStateStoreIsServing(t *testing.T) { + dir := t.TempDir() + db := filepath.Join(dir, "archie.db-tasks.sqlite") + out := backupPath(t, dir) + seedTask(t, openTaskStore(t, db), "before") + holdStoreOwnership(t, db) + if err := os.WriteFile(out, []byte("stale snapshot"), 0o600); err != nil { + t.Fatal(err) + } + + code, stdout, stderr := runRecoveryCmd(t, "backup", "-db", db, "-out", out) + if code != 0 { + t.Fatalf("backup exited %d: %s", code, stderr) + } + if !strings.Contains(stdout, out) || !strings.Contains(stdout, db) { + t.Errorf("backup must report the database and the snapshot it wrote; stdout = %q", stdout) + } + // The stale file was replaced by a real snapshot, and the snapshot is a + // store a restore can put back: same rows, readable through the store. + if got := taskTitles(t, out); len(got) != 1 || got[0] != "before" { + t.Fatalf("snapshot tasks = %v, want the one seeded task", got) + } + if entries, err := filepath.Glob(out + ".tmp*"); err != nil || len(entries) != 0 { + t.Errorf("backup left scratch files beside the snapshot: %v (%v)", entries, err) + } +} + +// The snapshot is the rollback state, so an interrupted backup must never +// leave a truncated file where a good snapshot was. +func TestRecoveryBackupRefusesAMissingDatabase(t *testing.T) { + dir := t.TempDir() + code, _, stderr := runRecoveryCmd(t, "backup", "-db", filepath.Join(dir, "absent.sqlite"), "-out", backupPath(t, dir)) + if code == 0 { + t.Fatal("backup of a missing database succeeded") + } + if !strings.Contains(stderr, "absent.sqlite") { + t.Errorf("refusal must name the database it could not read; stderr = %q", stderr) + } +} + +// Restoring is the forward-migration escape hatch: the previous release +// reads this file, so it must be exactly the snapshot, with the WAL that +// belongs to the replaced database gone rather than replayed over it. +func TestRecoveryRestorePutsTheSnapshotBackAndDropsItsWAL(t *testing.T) { + dir := t.TempDir() + db := filepath.Join(dir, "archie.db-tasks.sqlite") + out := backupPath(t, dir) + st := openTaskStore(t, db) + seedTask(t, st, "before") + backup(t, db, out) + seedTask(t, st, "after") + + code, stdout, stderr := runRecoveryCmd(t, "restore", "-db", db, "-from", out) + if code != 0 { + t.Fatalf("restore exited %d: %s", code, stderr) + } + if !strings.Contains(stdout, out) { + t.Errorf("restore must report the snapshot it restored from; stdout = %q", stdout) + } + // Checked before anything reopens the database: opening a store sets WAL + // mode, which creates these files, so the point is that the restore did not + // leave the replaced database's log behind -- not that a later reader + // cannot recreate its own. + for _, suffix := range []string{"-wal", "-shm"} { + if _, err := os.Stat(db + suffix); err == nil { + t.Errorf("%s survived the restore: the replaced database's write-ahead log must not be replayed over the snapshot", db+suffix) + } + } + if got := taskTitles(t, db); len(got) != 1 || got[0] != "before" { + t.Fatalf("restored tasks = %v, want only the snapshot's task", got) + } +} + +// Overwriting a database a running State Store still owns leaves that process +// writing to an unlinked inode while the store serves a file nobody owns -- +// the one way an offline command can destroy a task store. +func TestRecoveryRestoreRefusesAStoreTheStateStoreIsRunning(t *testing.T) { + dir := t.TempDir() + db := filepath.Join(dir, "archie.db-tasks.sqlite") + out := backupPath(t, dir) + st := openTaskStore(t, db) + seedTask(t, st, "before") + backup(t, db, out) + seedTask(t, st, "after") + holdStoreOwnership(t, db) + + code, _, stderr := runRecoveryCmd(t, "restore", "-db", db, "-from", out) + if code == 0 { + t.Fatal("restore against a live State Store succeeded") + } + if !strings.Contains(stderr, "owned by another process") { + t.Errorf("refusal must say the database is owned by another process; stderr = %q", stderr) + } + if got := taskTitles(t, db); len(got) != 2 { + t.Fatalf("refused restore changed the database: tasks = %v", got) + } +} + +func TestRecoveryRestoreRefusesAFileThatIsNotAStore(t *testing.T) { + dir := t.TempDir() + db := filepath.Join(dir, "archie.db-tasks.sqlite") + seedTask(t, openTaskStore(t, db), "live") + notAStore := filepath.Join(dir, "notes.txt") + if err := os.WriteFile(notAStore, []byte("this is not a database"), 0o600); err != nil { + t.Fatal(err) + } + + code, _, stderr := runRecoveryCmd(t, "restore", "-db", db, "-from", notAStore) + if code == 0 { + t.Fatal("restore from a file that is not a store succeeded") + } + if !strings.Contains(stderr, notAStore) { + t.Errorf("refusal must name the snapshot it could not read; stderr = %q", stderr) + } + if got := taskTitles(t, db); len(got) != 1 { + t.Fatalf("refused restore changed the database: tasks = %v", got) + } +} + +func TestRecoveryRestoreRefusesTheDatabaseAsItsOwnSnapshot(t *testing.T) { + // Restoring a database from itself would delete the write-ahead log of the + // file it is about to copy, so the one case that must never reach the swap + // is refused up front. + db := filepath.Join(t.TempDir(), "archie.db-tasks.sqlite") + seedTask(t, openTaskStore(t, db), "live") + + code, _, stderr := runRecoveryCmd(t, "restore", "-db", db, "-from", db) + if code == 0 { + t.Fatal("restore from the database itself succeeded") + } + if !strings.Contains(stderr, "is the database itself") { + t.Errorf("refusal must say the snapshot is the database; stderr = %q", stderr) + } + if got := taskTitles(t, db); len(got) != 1 { + t.Fatalf("refused restore changed the database: tasks = %v", got) + } +} + +func TestRecoveryValidateAcceptsAHealthyStore(t *testing.T) { + dir := t.TempDir() + db := filepath.Join(dir, "archie.db-tasks.sqlite") + st := openTaskStore(t, db) + seedTask(t, st, "healthy") + seedResource(t, st, controlplane.ModelRoleAssignmentsKind, `{"implement":"anthropic/claude"}`, "seed") + + code, stdout, stderr := runRecoveryCmd(t, "validate", "-db", db) + if code != 0 { + t.Fatalf("validate exited %d: %s", code, stderr) + } + if !strings.Contains(stdout, "schema version") { + t.Errorf("validate must report the schema version it verified; stdout = %q", stdout) + } +} + +// validate exists to answer "would archied start against this file". Each case +// below is a state the daemon fails closed on, or a file it cannot read at all. +func TestRecoveryValidateRefusesWhatTheDaemonRefuses(t *testing.T) { + newStore := func(t *testing.T) string { + t.Helper() + db := filepath.Join(t.TempDir(), "archie.db-tasks.sqlite") + openTaskStore(t, db) + return db + } + + t.Run("forward_migrated_schema", func(t *testing.T) { + // A database written by a newer release is the state a rollback has to + // recognise before it hands the file to an older binary. + db := newStore(t) + if err := execRaw(t, db, `PRAGMA user_version = 99`); err != nil { + t.Fatal(err) + } + code, _, stderr := runRecoveryCmd(t, "validate", "-db", db) + if code == 0 { + t.Fatal("validate accepted a forward-migrated database") + } + if !strings.Contains(stderr, "newer than supported version") { + t.Errorf("refusal must name the schema version; stderr = %q", stderr) + } + }) + + t.Run("corrupt_file", func(t *testing.T) { + // A real database header over pages that are not a database. The magic + // is right, so this fails on the content rather than on the format, which + // is the shape a half-written or truncated store has. + db := filepath.Join(t.TempDir(), "archie.db-tasks.sqlite") + corrupt := append([]byte("SQLite format 3\x00"), bytes.Repeat([]byte{0xff}, 4096)...) + if err := os.WriteFile(db, corrupt, 0o600); err != nil { + t.Fatal(err) + } + code, _, stderr := runRecoveryCmd(t, "validate", "-db", db) + if code == 0 { + t.Fatal("validate accepted a corrupt database") + } + if !strings.Contains(stderr, db) { + t.Errorf("refusal must name the file; stderr = %q", stderr) + } + }) + + t.Run("not_a_database", func(t *testing.T) { + // Pointing -db at the wrong file is the common operator mistake, and the + // configured db_path is a different file from the task database the + // State Store owns. + db := filepath.Join(t.TempDir(), "archie.db") + if err := os.WriteFile(db, []byte("db_path, not the task database"), 0o600); err != nil { + t.Fatal(err) + } + code, _, stderr := runRecoveryCmd(t, "validate", "-db", db) + if code == 0 { + t.Fatal("validate accepted a file that is not a database") + } + if !strings.Contains(stderr, db) { + t.Errorf("refusal must name the file; stderr = %q", stderr) + } + }) + + t.Run("stored_resource_that_will_not_validate", func(t *testing.T) { + // The stored settings value that stops archied starting: it can only + // get here by bypassing the write path, which is exactly the state the + // control plane fails closed on and the operator has no path back from. + db := newStore(t) + seedResource(t, openTaskStore(t, db), controlplane.ModelRoleAssignmentsKind, `{"implement":"claude"}`, "hand-edited") + code, _, stderr := runRecoveryCmd(t, "validate", "-db", db) + if code == 0 { + t.Fatal("validate accepted a stored resource the write path would refuse") + } + if !strings.Contains(stderr, controlplane.ModelRoleAssignmentsKind) { + t.Errorf("refusal must name the resource kind; stderr = %q", stderr) + } + }) + + t.Run("missing_database", func(t *testing.T) { + db := newStore(t) + if err := os.Remove(db); err != nil { + t.Fatal(err) + } + code, _, stderr := runRecoveryCmd(t, "validate", "-db", db) + if code == 0 { + t.Fatal("validate accepted a database that does not exist") + } + if !strings.Contains(stderr, db) { + t.Errorf("refusal must name the file; stderr = %q", stderr) + } + }) + + t.Run("store_without_the_control_plane_table", func(t *testing.T) { + // A store written before the control plane existed holds no stored + // settings. The serving process creates the table on its next start, so + // refusing this file would send an operator to restore a snapshot for a + // database that would have started perfectly well. + db := newStore(t) + if err := execRaw(t, db, `DROP TABLE resources`); err != nil { + t.Fatal(err) + } + code, stdout, stderr := runRecoveryCmd(t, "validate", "-db", db) + if code != 0 { + t.Fatalf("validate exited %d: %s", code, stderr) + } + if !strings.Contains(stdout, "0 stored resources validate") { + t.Errorf("validate must report that there was nothing stored to check; stdout = %q", stdout) + } + }) +} + +// rollback is the one operation that closes the documented gap: with the State +// Store stopped, it replays the value an earlier revision recorded through the +// ordinary replace, so no rollback RPC is needed and the rollback itself is +// audited as one more revision. +func TestRecoveryRollbackReplaysTheRevisionThroughReplace(t *testing.T) { + dir := t.TempDir() + db := filepath.Join(dir, "archie.db-tasks.sqlite") + st := openTaskStore(t, db) + seedResource(t, st, controlplane.ModelRoleAssignmentsKind, `{"implement":"anthropic/claude"}`, "first") + seedResource(t, st, controlplane.ModelRoleAssignmentsKind, `{"implement":"openai/gpt"}`, "second") + + code, stdout, stderr := runRecoveryCmd(t, "rollback", "-db", db, "-kind", controlplane.ModelRoleAssignmentsKind) + if code != 0 { + t.Fatalf("rollback exited %d: %s", code, stderr) + } + if !strings.Contains(stdout, controlplane.ModelRoleAssignmentsKind) { + t.Errorf("rollback must name the resource it replayed; stdout = %q", stdout) + } + resource, err := st.Resource(t.Context(), controlplane.ModelRoleAssignmentsKind) + if err != nil { + t.Fatal(err) + } + if string(resource.Value) != `{"implement":"anthropic/claude"}` { + t.Errorf("resource value = %s, want the first revision's value", resource.Value) + } + if resource.Version != 3 { + t.Errorf("resource version = %d, want a new revision (3) rather than a rewrite of the old one", resource.Version) + } + history, err := st.ResourceHistory(t.Context(), controlplane.ModelRoleAssignmentsKind, 0) + if err != nil { + t.Fatal(err) + } + if len(history) != 3 { + t.Fatalf("history has %d revisions, want the rollback recorded as the third", len(history)) + } + if history[0].Source != archied.OfflineRollbackSource { + t.Errorf("newest revision source = %q, want %q so the rollback is auditable", history[0].Source, archied.OfflineRollbackSource) + } +} + +// The revision to restore may be named, which is how an operator reaches past +// the most recent change without replaying it first. +func TestRecoveryRollbackRestoresANamedRevision(t *testing.T) { + dir := t.TempDir() + db := filepath.Join(dir, "archie.db-tasks.sqlite") + st := openTaskStore(t, db) + seedResource(t, st, controlplane.ModelRoleAssignmentsKind, `{"implement":"anthropic/claude"}`, "first") + seedResource(t, st, controlplane.ModelRoleAssignmentsKind, `{"implement":"openai/gpt"}`, "second") + seedResource(t, st, controlplane.ModelRoleAssignmentsKind, `{"implement":"google/gemini"}`, "third") + + code, _, stderr := runRecoveryCmd(t, "rollback", "-db", db, "-kind", controlplane.ModelRoleAssignmentsKind, "-revision", "1") + if code != 0 { + t.Fatalf("rollback exited %d: %s", code, stderr) + } + resource, err := st.Resource(t.Context(), controlplane.ModelRoleAssignmentsKind) + if err != nil { + t.Fatal(err) + } + if string(resource.Value) != `{"implement":"anthropic/claude"}` { + t.Errorf("resource value = %s, want the value recorded at revision 1", resource.Value) + } +} + +func TestRecoveryRollbackRefusesWhatItCannotReplay(t *testing.T) { + newStore := func(t *testing.T, revisions int) string { + t.Helper() + db := filepath.Join(t.TempDir(), "archie.db-tasks.sqlite") + st := openTaskStore(t, db) + for i := range revisions { + value := `{"implement":"anthropic/claude"}` + if i > 0 { + value = `{"implement":"openai/gpt"}` + } + seedResource(t, st, controlplane.ModelRoleAssignmentsKind, value, "seed") + } + return db + } + + t.Run("unknown_kind", func(t *testing.T) { + db := newStore(t, 2) + code, _, stderr := runRecoveryCmd(t, "rollback", "-db", db, "-kind", "not-a-resource") + if code == 0 { + t.Fatal("rollback of an unknown resource kind succeeded") + } + if !strings.Contains(stderr, "not-a-resource") { + t.Errorf("refusal must name the kind; stderr = %q", stderr) + } + }) + + t.Run("no_earlier_revision", func(t *testing.T) { + db := newStore(t, 1) + code, _, stderr := runRecoveryCmd(t, "rollback", "-db", db, "-kind", controlplane.ModelRoleAssignmentsKind) + if code == 0 { + t.Fatal("rollback with no earlier revision succeeded") + } + if !strings.Contains(stderr, "no earlier revision") { + t.Errorf("refusal must say there is nothing to roll back to; stderr = %q", stderr) + } + }) + + t.Run("current_revision", func(t *testing.T) { + db := newStore(t, 2) + code, _, stderr := runRecoveryCmd(t, "rollback", "-db", db, "-kind", controlplane.ModelRoleAssignmentsKind, "-revision", "2") + if code == 0 { + t.Fatal("rollback of the current revision succeeded") + } + if !strings.Contains(stderr, "is the current version") { + t.Errorf("refusal must say the revision is already the current value; stderr = %q", stderr) + } + }) + + t.Run("live_state_store", func(t *testing.T) { + db := newStore(t, 2) + holdStoreOwnership(t, db) + code, _, stderr := runRecoveryCmd(t, "rollback", "-db", db, "-kind", controlplane.ModelRoleAssignmentsKind) + if code == 0 { + t.Fatal("rollback against a live State Store succeeded") + } + if !strings.Contains(stderr, "owned by another process") { + t.Errorf("refusal must say the database is owned by another process; stderr = %q", stderr) + } + }) +} + +// A positional argument that is not a subcommand used to be ignored, leaving +// an operator who mistyped a command watching a server start instead. +func TestRecoveryRejectsAnUnknownSubcommand(t *testing.T) { + var stdout, stderr bytes.Buffer + code := runRecovery([]string{"backp", "-db", "x"}, &stdout, &stderr) + if code == 0 { + t.Fatal("an unknown subcommand exited 0") + } + if !strings.Contains(stderr.String(), "backp") { + t.Errorf("refusal must name the unknown subcommand; stderr = %q", stderr.String()) + } +} + +// seedResource writes one revision of a control-plane resource through the +// store, the way the control plane's replace does. The request ID is derived +// from the label and the revision it produces: the store deduplicates a +// repeated request ID, so a fixed one would make the second write of a series +// a silent no-op. +func seedResource(t *testing.T, st *store.Store, kind, value, label string) { + t.Helper() + current, err := st.Resource(t.Context(), kind) + expected := int64(0) + if err == nil { + expected = current.Version + } + var document any + if err := json.Unmarshal([]byte(value), &document); err != nil { + t.Fatal(err) + } + encoded, err := json.Marshal(document) + if err != nil { + t.Fatal(err) + } + if _, err := st.PutResource(t.Context(), store.ResourceWrite{ + Kind: kind, Value: encoded, Actor: label, Source: "test", + RequestID: fmt.Sprintf("%s-%d", label, expected+1), ExpectedVersion: expected, + }); err != nil { + t.Fatalf("seed resource %s: %v", kind, err) + } +} + +// execRaw runs one statement against the database file the way an operator +// editing a broken store would, bypassing every check the store applies. +func execRaw(t *testing.T, path, statement string) error { + t.Helper() + db, err := sql.Open("sqlite", path) + if err != nil { + return err + } + defer func() { _ = db.Close() }() + _, err = db.ExecContext(t.Context(), statement) + return err +} diff --git a/docs/architecture/safe-change-and-recovery.md b/docs/architecture/safe-change-and-recovery.md index 564e27e00..6f10f70ff 100644 --- a/docs/architecture/safe-change-and-recovery.md +++ b/docs/architecture/safe-change-and-recovery.md @@ -120,21 +120,40 @@ file values would restore the second source of truth the control plane exists to remove. The cost is that a bad stored value stops the daemon starting, and the value can only be corrected through the daemon's own API. -### Recovery: not yet built +### Recovery: offline commands on the State Store -There is no escape hatch. Nothing skips the control plane at boot, and the only -writer for a resource is `ControlPlaneService.Command` on a running State Store. -An operator whose stored settings will not validate has no supported path back -short of editing `archie.db` by hand. +Nothing skips the control plane at boot, and the only writer for a resource is +`ControlPlaneService.Command` on a running State Store -- which is exactly what +the daemon fails closed without. `archie-state-store` therefore carries the +offline recovery commands (`archie-core-6zw0`). They work on the task database +file directly -- the configured `db_path` with `-tasks.sqlite` appended, not +`db_path` itself -- and need neither `archied` nor the Web UI. -`archie-core-6zw0` is the missing piece: offline commands that back up, restore, -validate and roll back the database without `archied` or the Web UI. Until it -lands, treat a settings change that requires a restart as a change that can -prevent one, and take a copy of `archie.db` first. +| Command | What it does | State Store | +| --- | --- | --- | +| `backup -db F -out F` | writes a transactionally consistent snapshot (`VACUUM INTO`), verified as a usable store before it is published | may be running | +| `restore -db F -from F` | replaces the database with a snapshot and drops the replaced file's `-wal`/`-shm` | must be stopped | +| `validate -db F` | checks the file the way boot meets it: readable, not corrupt, schema no newer than the binary, and every stored resource decoding under the definition that owns it | must be stopped | +| `rollback -db F -kind K [-revision N]` | replays an earlier revision of a stored resource through the ordinary replace, recording the rollback as a new revision | must be stopped | -Resource history is the in-band remedy and does exist: every write records its -prior value, so an edit made through the dashboard or a channel can be restored -by replaying an earlier revision while the State Store is up. +No rollback RPC exists and none is needed: a revision carries its own value, so +restoring one is the same replace the dashboard and the channels already use. +The offline command is that operation with the store stopped -- the one state in +which the daemon's own API cannot serve it. + +"Must be stopped" is checked rather than assumed. The serving process takes an +exclusive flock on `.lock` for its whole life, and `restore` and `rollback` +refuse with `task database is owned by another process` while another process +holds it. `backup` is the deliberate exception: the update installer runs inside +the process it is replacing and cannot stop the State Store, so the snapshot +that makes a failed update reversible is taken against a serving store -- and +`scripts/archie-update-install` takes it through this subcommand rather than +through its own `sqlite3` shell, so the installer and an operator run one +implementation. + +`rollback` is the surgical route back from a bad stored value; `restore` is the +blunt one, and is also the way back from a schema migration a release cannot +grow out of. ## Gateway restart: two constraints learned the hard way diff --git a/internal/app/archied/state_store.go b/internal/app/archied/state_store.go index d2a1da380..b1b919d35 100644 --- a/internal/app/archied/state_store.go +++ b/internal/app/archied/state_store.go @@ -215,7 +215,25 @@ func (b *boot) openStateStore(ctx context.Context) error { log.Error("configure bindings cipher", "err", err) return err } - st, err := openProductionTaskStore(ctx, taskDBPath(cfg.DBPath), store.WithBindingCipher(bindingCipher)) + // Claim the database before opening it. This process is the file's owner + // for as long as it lives, and the claim is what an offline recovery + // command checks before it rewrites the file: without it, "the State Store + // is stopped" is a rule the operator is trusted to have followed rather + // than a fact the command can verify. A second State Store on the same + // database fails here, which is the single-owner invariant stated instead + // of merely documented. + path := taskDBPath(cfg.DBPath) + ownership, err := store.AcquireOwnership(path) + if err != nil { + log.Error("claim state store ownership", "err", err) + return err + } + b.addCleanup(func() { + if err := ownership.Release(); err != nil { + log.Error("release state store ownership", "err", err) + } + }) + st, err := openProductionTaskStore(ctx, path, store.WithBindingCipher(bindingCipher)) if err != nil { log.Error("open state store", "err", err) return err diff --git a/internal/app/archied/state_store_recovery.go b/internal/app/archied/state_store_recovery.go new file mode 100644 index 000000000..bf0acc93d --- /dev/null +++ b/internal/app/archied/state_store_recovery.go @@ -0,0 +1,210 @@ +// state_store_recovery.go composes the offline recovery subcommands of the +// standalone archie-state-store process. They exist because the control plane +// fails closed: a stored setting that will not validate stops archied +// starting, and the in-band remedy -- replaying an earlier revision while the +// State Store is up -- is unavailable in exactly the case that needs it +// (docs/architecture/safe-change-and-recovery.md). +// +// Every operation works on the task database file directly, with the State +// Store stopped, and the ones that write refuse when another process owns the +// file. Backup is the exception and deliberately so: the update installer +// cannot stop the process that runs it, so the snapshot that makes a failed +// update reversible is taken against a serving store through SQLite's own +// consistent copy. +package archied + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/samcharles93/archie-core/internal/app/controlplane" + pb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1" + "github.com/samcharles93/archie-core/internal/store" +) + +// The audit identity every offline rollback records. A rollback is one more +// revision of the resource, written through the ordinary replace, so the +// operator can see who went back and when -- and can go forward again the same +// way, with the State Store up or down. +const ( + OfflineRollbackSource = "archie-state-store rollback" + OfflineRollbackActor = "operator:offline-recovery" +) + +// Recovery operations, as named on the command line. +const ( + RecoveryBackup = "backup" + RecoveryRestore = "restore" + RecoveryValidate = "validate" + RecoveryRollback = "rollback" +) + +// StateStoreRecoveryOptions are the process inputs for one offline recovery +// operation. +type StateStoreRecoveryOptions struct { + // Operation is one of backup, restore, validate, rollback. + Operation string + // DB is the task database file the State Store owns: the configured + // db_path with "-tasks.sqlite" appended, not the configured path itself. + DB string + // Out is the snapshot backup writes. + Out string + // From is the snapshot restore reads. + From string + // Kind is the control-plane resource kind rollback replays. + Kind string + // Revision is the revision rollback replays. Zero means the newest + // revision older than the one the resource carries now. + Revision int64 +} + +// RunStateStoreRecovery performs one offline operation on the task database and +// returns the line the command reports on stdout. +func RunStateStoreRecovery(ctx context.Context, options StateStoreRecoveryOptions) (string, error) { + switch options.Operation { + case RecoveryBackup: + if options.DB == "" || options.Out == "" { + return "", errors.New("backup requires -db and -out") + } + if err := store.Backup(ctx, options.DB, options.Out); err != nil { + return "", err + } + return fmt.Sprintf("backed up %s to %s", options.DB, options.Out), nil + case RecoveryRestore: + if options.DB == "" || options.From == "" { + return "", errors.New("restore requires -db and -from") + } + if err := store.Restore(ctx, options.DB, options.From); err != nil { + return "", err + } + return fmt.Sprintf("restored %s from %s; start the State Store again", options.DB, options.From), nil + case RecoveryValidate: + if options.DB == "" { + return "", errors.New("validate requires -db") + } + return validateStore(ctx, options.DB) + case RecoveryRollback: + if options.DB == "" || options.Kind == "" { + return "", errors.New("rollback requires -db and -kind") + } + return rollbackResource(ctx, options) + default: + return "", fmt.Errorf("unknown recovery command %q", options.Operation) + } +} + +// validateStore answers "would archied start against this file". The file-level +// checks come first, so the store is only read once its schema is one this +// binary understands; the resource check then decodes every stored value with +// the definition that owns it, which is the same validation a write applies. +func validateStore(ctx context.Context, path string) (string, error) { + version, err := store.ValidateFile(ctx, path) + if err != nil { + return "", err + } + st, err := store.OpenReadOnly(ctx, path) + if err != nil { + return "", err + } + defer func() { _ = st.Close() }() + // The resources table is only absent from a store written before the control + // plane existed. That store holds no settings to validate, and the serving + // process creates the table on its next start -- refusing it here would send + // an operator to restore a snapshot for nothing. + checked := 0 + stored, err := st.StoresResources(ctx) + if err != nil { + return "", err + } + if stored { + checked, err = controlplane.NewServer(st).ValidateStored(ctx) + if err != nil { + return "", err + } + } + return fmt.Sprintf("%s is a valid store at schema version %d; %d stored resources validate", path, version, checked), nil +} + +// rollbackResource is the one operation the Web UI cannot be asked to perform +// here: the daemon fails closed on the bad value, so the store that would serve +// the rollback is the store that will not start. It replays the revision's own +// value through the ordinary replace, so no rollback RPC exists or is needed, +// and the rollback is audited as one more revision. +func rollbackResource(ctx context.Context, options StateStoreRecoveryOptions) (string, error) { + // Before the lock and before the open: opening a store creates the file, + // and a rollback pointed at a mistyped path must not leave an empty + // database where the operator expected theirs. + if err := store.RequireDatabase(options.DB); err != nil { + return "", err + } + ownership, err := store.AcquireOwnership(options.DB) + if err != nil { + return "", err + } + defer func() { _ = ownership.Release() }() + + st, err := store.Open(ctx, options.DB) + if err != nil { + return "", err + } + defer func() { _ = st.Close() }() + + server := controlplane.NewServer(st) + if !server.Owns(options.Kind) { + return "", fmt.Errorf("unknown resource kind %q", options.Kind) + } + current, err := st.Resource(ctx, options.Kind) + if errors.Is(err, store.ErrResourceNotFound) { + return "", fmt.Errorf("%s has no stored value to roll back", options.Kind) + } + if err != nil { + return "", fmt.Errorf("read %s: %w", options.Kind, err) + } + revision, err := revisionToReplay(ctx, st, options.Kind, current.Version, options.Revision) + if err != nil { + return "", err + } + replaced, err := server.Command(ctx, &pb.CommandRequest{ + Kind: options.Kind, + Command: "replace", + ValueJson: revision.Value, + ExpectedVersion: current.Version, + Actor: OfflineRollbackActor, + Source: OfflineRollbackSource, + RequestId: fmt.Sprintf("recovery-rollback-%s-%d", options.Kind, time.Now().UTC().UnixNano()), + }) + if err != nil { + return "", fmt.Errorf("replay %s at revision %d: %w", options.Kind, revision.Version, err) + } + return fmt.Sprintf("rolled back %s from version %d to the value recorded at version %d; the store now holds version %d", + options.Kind, current.Version, revision.Version, replaced.GetResource().GetVersion()), nil +} + +// revisionToReplay picks the value to put back: the revision the operator +// named, or the newest one older than the value the resource carries now. +func revisionToReplay(ctx context.Context, st *store.Store, kind string, current, requested int64) (store.Resource, error) { + history, err := st.ResourceHistory(ctx, kind, 0) + if err != nil { + return store.Resource{}, fmt.Errorf("read %s history: %w", kind, err) + } + if requested > 0 { + for _, revision := range history { + if revision.Version != requested { + continue + } + if revision.Version == current { + return store.Resource{}, fmt.Errorf("%s revision %d is the current version; there is nothing to roll back to", kind, requested) + } + return revision, nil + } + return store.Resource{}, fmt.Errorf("%s has no revision %d", kind, requested) + } + for _, revision := range history { + if revision.Version < current { + return revision, nil + } + } + return store.Resource{}, fmt.Errorf("%s has no earlier revision to roll back to", kind) +} diff --git a/internal/app/controlplane/registry.go b/internal/app/controlplane/registry.go index b49186ce9..ddf66142a 100644 --- a/internal/app/controlplane/registry.go +++ b/internal/app/controlplane/registry.go @@ -89,6 +89,42 @@ func (s *Server) ImportConfig(ctx context.Context, cfg config.Config) (map[strin return versions, nil } +// Owns reports whether kind is a resource the control plane owns, so a caller +// acting on a stored value can refuse a kind nobody defines before it reaches +// the store. +func (s *Server) Owns(kind string) bool { + _, ok := s.definitions[kind] + return ok +} + +// ValidateStored decodes every stored resource with the definition that owns +// it, using the same Decode the write path applies, and reports how many +// resources it checked. Errors name the kind and the revision it was stored at, +// which is what an operator needs to roll the value back. +// +// This is how an offline validate reproduces the refusal that stops archied +// starting: the check is the writer's own, so it cannot drift from what a +// replace would accept, and a value can only be stored unvalidated by bypassing +// the write path entirely. +func (s *Server) ValidateStored(ctx context.Context) (int, error) { + checked := 0 + var failures []error + for _, definition := range s.ordered { + resource, err := s.store.Resource(ctx, definition.Kind) + if errors.Is(err, store.ErrResourceNotFound) { + continue + } + if err != nil { + return checked, fmt.Errorf("read %s: %w", definition.Kind, err) + } + checked++ + if _, err := definition.Decode(resource.Value); err != nil { + failures = append(failures, fmt.Errorf("%s at version %d: %w", definition.Kind, resource.Version, err)) + } + } + return checked, errors.Join(failures...) +} + func validateAs[T any](input []byte, validate func(T) error) error { var value T decoder := json.NewDecoder(bytes.NewReader(input)) diff --git a/internal/releaseupdate/install_script_test.go b/internal/releaseupdate/install_script_test.go index fdae8fac4..77f58e27e 100644 --- a/internal/releaseupdate/install_script_test.go +++ b/internal/releaseupdate/install_script_test.go @@ -382,6 +382,13 @@ esac `) writeFakeCommand(t, fakeDir, "go", ` printf '%s\n' "go $*" >> "$ARCHIE_TEST_CALLS" +# The pre-flight task-database backup runs through the release's own +# archie-state-store subcommand, so a release whose recovery path cannot run +# must refuse the update. +if [ "$1" = "run" ]; then + [ "${ARCHIE_TEST_GO_FAILS_RUN:-}" = "1" ] && exit 1 + exit 0 +fi out="" while [ "$#" -gt 0 ]; do if [ "$1" = "-o" ]; then out="$2"; break; fi @@ -745,6 +752,10 @@ func TestUpdateWatchdogCyclesEveryUnitInDependencyOrder(t *testing.T) { // value up produces an empty backup, and a rollback then restores that empty // file over the database and deletes its -wal/-shm. A rollback that destroys // the task store is worse than no rollback. +// +// The snapshot is taken by the release's own recovery subcommand rather than +// by hand-rolled shell, so the installer and an operator run the same code and +// the two cannot drift apart. func TestUpdateInstallBacksUpTheSiblingTaskDatabase(t *testing.T) { work := t.TempDir() configured := filepath.Join(work, "archie.db") @@ -764,7 +775,8 @@ func TestUpdateInstallBacksUpTheSiblingTaskDatabase(t *testing.T) { "ARCHIE_TASK_DB_BACKUP": backup, }, "db_path = \""+configured+"\"\n\n[containers]\nimage = 'registry.example/archie-agent:stable'\n", false) - assertCallContains(t, calls, "sqlite3", realStore, ".backup") + assertCallContains(t, calls, "go run ./cmd/archie-state-store backup", "-db "+realStore, "-out "+backup) + assertCallAbsent(t, calls, "sqlite3") assertCallContains(t, calls, "systemd-run", "--setenv=ARCHIE_TASK_DB_PATH="+realStore) // The zero-byte placeholder is not the store, and must never be the thing // handed to the watchdog as ARCHIE_TASK_DB_PATH. @@ -793,9 +805,9 @@ func TestUpdateInstallIgnoresDbPathFromOtherSections(t *testing.T) { "ARCHIE_TASK_DB_BACKUP": backup, }, "db_path = \""+configured+"\"\n\n[indexing]\ndb_path = \""+indexStore+"\"\n\n[containers]\nimage = 'registry.example/archie-agent:stable'\n", false) - assertCallContains(t, calls, "sqlite3", realStore) + assertCallContains(t, calls, "go run ./cmd/archie-state-store backup", "-db "+realStore) for _, call := range calls { - if strings.Contains(call, "sqlite3") && strings.Contains(call, indexStore) { + if strings.Contains(call, "go run ./cmd/archie-state-store backup") && strings.Contains(call, indexStore) { t.Errorf("backed up the indexing database %s: %q", indexStore, call) } } @@ -815,7 +827,7 @@ func TestUpdateInstallRefusesWhenTaskDatabaseCannotBeBackedUp(t *testing.T) { "ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0", "ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0", "ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0", - "ARCHIE_TEST_SQLITE3_FAILS": "1", + "ARCHIE_TEST_GO_FAILS_RUN": "1", }, "db_path = \""+configured+"\"\n\n[containers]\nimage = 'registry.example/archie-agent:stable'\n", true) if !strings.Contains(output, "task database") { diff --git a/internal/store/maintenance.go b/internal/store/maintenance.go new file mode 100644 index 000000000..12c3af029 --- /dev/null +++ b/internal/store/maintenance.go @@ -0,0 +1,253 @@ +package store + +import ( + "context" + "database/sql" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" +) + +// This file is the offline half of the store's contract with its owner: the +// operations the State Store process cannot perform on itself, because they +// read or replace the very file it has open. They are what +// docs/prds/runtime-control-plane.md ("Bootstrap, migration, and recovery") +// asks for -- back up, restore and validate the database without archied or +// the Web UI -- and every one of them works on the file directly. +// +// The read-only operations never take the ownership lock, because they are +// what an operator runs against a serving store. The one that replaces the +// file does, because a rewrite under a running server is the only way an +// offline command can destroy a task store. + +// OpenReadOnly opens an existing store for reading only: no schema is created +// and no migration runs. That matters for recovery -- the snapshot an update +// takes has to stay exactly what the release that wrote it expects, and +// migrating a copy during verification would hand a rollback a schema its +// binary refuses. +func OpenReadOnly(ctx context.Context, path string) (*Store, error) { + if err := RequireDatabase(path); err != nil { + return nil, err + } + db, err := sql.Open("sqlite", readonlyDSN(path)) + if err != nil { + return nil, err + } + db.SetMaxOpenConns(1) + // sql.Open is lazy: the first statement is where a file that is not a + // database, a missing WAL sidecar or an unreadable path surfaces. + var tables int + if err := db.QueryRowContext(ctx, `SELECT count(*) FROM sqlite_master`).Scan(&tables); err != nil { + return nil, errors.Join(fmt.Errorf("read task database %s: %w", path, err), db.Close()) + } + return &Store{db: db}, nil +} + +// readonlyDSN reads the store without the write-ahead log pragma the serving +// process sets: a reader has no business changing the database's journal mode, +// and the read-only flag makes an accidental write an error rather than a +// silent modification of a file someone else owns. +func readonlyDSN(path string) string { + separator := "?" + if strings.HasSuffix(path, "?") || strings.HasSuffix(path, "&") { + separator = "" + } else if strings.Contains(path, "?") { + separator = "&" + } + return path + separator + "mode=ro&_pragma=busy_timeout(5000)" +} + +// StoresResources reports whether the store's file carries the control-plane +// resources table. A store written before the control plane existed has none +// and holds no stored settings to validate; the serving process creates the +// table on its next start, so an offline check must tell that apart from +// resources it cannot read. +func (s *Store) StoresResources(ctx context.Context) (bool, error) { + var tables int + if err := s.db.QueryRowContext(ctx, `SELECT count(*) FROM sqlite_master WHERE type='table' AND name='resources'`).Scan(&tables); err != nil { + return false, fmt.Errorf("inspect store schema: %w", err) + } + return tables > 0, nil +} + +// RequireDatabase reports whether path is an existing store file. A writer +// that must not create the file it fails to find asks this first: a recovery +// command pointed at a mistyped path has to refuse, not leave a fresh empty +// database where an operator expected theirs. +func RequireDatabase(path string) error { + if _, err := os.Stat(path); err != nil { + return fmt.Errorf("task database %s: %w", path, err) + } + return nil +} + +// ValidateFile checks an existing store the way the serving process meets it +// on startup and reports the schema version it carries: the file is a SQLite +// database this binary can read, it is not corrupt, and its schema is not +// newer than this binary supports. +func ValidateFile(ctx context.Context, path string) (int, error) { + st, err := OpenReadOnly(ctx, path) + if err != nil { + return 0, err + } + defer func() { _ = st.Close() }() + return st.check(ctx) +} + +// check runs the file-level checks. integrity_check returns one row per +// problem, or the single row "ok"; reading the first row is therefore the +// verdict rather than a sample. +func (s *Store) check(ctx context.Context) (int, error) { + var result string + if err := s.db.QueryRowContext(ctx, `PRAGMA integrity_check`).Scan(&result); err != nil { + return 0, fmt.Errorf("integrity check: %w", err) + } + if result != "ok" { + return 0, fmt.Errorf("integrity check: %s", result) + } + var version int + if err := s.db.QueryRowContext(ctx, `PRAGMA user_version`).Scan(&version); err != nil { + return 0, fmt.Errorf("read schema version: %w", err) + } + if version > taskSchemaVersion { + return 0, fmt.Errorf("database schema version %d is newer than supported version %d", version, taskSchemaVersion) + } + return version, nil +} + +// Backup writes a transactionally consistent snapshot of database to snapshot, +// using SQLite's own copy, so a store that is being written to cannot produce +// a torn file. +// +// It deliberately does not take the ownership lock. The update installer runs +// inside the process it is updating and cannot stop the State Store, so the +// snapshot that makes a failed update reversible is taken against a serving +// store; a backup that insisted on an exclusively owned file would break the +// one caller that exists. The snapshot is written beside the destination and +// renamed into place, so an interrupted backup never replaces a snapshot that +// was good with one that is not. +func Backup(ctx context.Context, database, snapshot string) error { + if err := RequireDatabase(database); err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(snapshot), 0o755); err != nil { + return fmt.Errorf("create snapshot directory: %w", err) + } + db, err := sql.Open("sqlite", sqliteDSN(database)) + if err != nil { + return fmt.Errorf("open task database %s: %w", database, err) + } + defer func() { _ = db.Close() }() + + staging := snapshot + ".tmp" + if err := os.Remove(staging); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("clear staging snapshot %s: %w", staging, err) + } + if _, err := db.ExecContext(ctx, `VACUUM INTO ?`, staging); err != nil { + return errors.Join(fmt.Errorf("snapshot %s: %w", database, err), os.Remove(staging)) + } + // A snapshot that is not itself a usable store is worse than no snapshot: + // the update path treats its existence as proof that a rollback is + // possible. + if _, err := ValidateFile(ctx, staging); err != nil { + return errors.Join(fmt.Errorf("snapshot is not a usable store: %w", err), os.Remove(staging)) + } + if err := os.Rename(staging, snapshot); err != nil { + return errors.Join(fmt.Errorf("publish snapshot %s: %w", snapshot, err), os.Remove(staging)) + } + return nil +} + +// Restore replaces database with the snapshot at snapshot. The snapshot is +// verified before anything is destroyed, so a file that is not a usable store +// cannot take the database down with it, and the database is replaced by a +// rename, so a failure leaves either the old file or the new one rather than a +// mixture. +// +// It holds the ownership lock for its duration: rewriting a database a running +// State Store still has open would leave that process writing to an unlinked +// inode while the store served a file nobody owns. +func Restore(ctx context.Context, database, snapshot string) error { + if _, err := os.Stat(filepath.Dir(database)); err != nil { + return fmt.Errorf("restore destination %s: %w", database, err) + } + if same, err := sameFile(database, snapshot); err != nil { + return err + } else if same { + return fmt.Errorf("restore snapshot %s is the database itself", snapshot) + } + ownership, err := AcquireOwnership(database) + if err != nil { + return err + } + defer func() { _ = ownership.Release() }() + + if _, err := ValidateFile(ctx, snapshot); err != nil { + return fmt.Errorf("snapshot is not a usable store: %w", err) + } + staging := database + ".restore.tmp" + if err := copyFile(snapshot, staging); err != nil { + return errors.Join(err, os.Remove(staging)) + } // The replaced database's write-ahead log belongs to the file that is + // going away. It goes first: between the two calls the database is merely + // incomplete, whereas a WAL left beside the file it does not describe is + // the state SQLite cannot recover from. + for _, sidecar := range []string{"-wal", "-shm"} { + if err := os.Remove(database + sidecar); err != nil && !os.IsNotExist(err) { + return errors.Join(fmt.Errorf("remove %s%s: %w", database, sidecar, err), os.Remove(staging)) + } + } + if err := os.Rename(staging, database); err != nil { + return errors.Join(fmt.Errorf("publish restored database %s: %w", database, err), os.Remove(staging)) + } + return nil +} + +func sameFile(a, b string) (bool, error) { + first, err := os.Stat(a) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + second, err := os.Stat(b) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + return os.SameFile(first, second), nil +} + +// copyFile writes source over a new file at path, flushed to disk before it is +// returned: the rename that publishes it must not reach the directory entries +// ahead of the bytes they point at. The snapshot's own permissions are kept, so +// a restore does not silently change who may read the operator's database. +func copyFile(source, path string) (retErr error) { + info, err := os.Stat(source) + if err != nil { + return fmt.Errorf("stat snapshot %s: %w", source, err) + } + in, err := os.Open(source) + if err != nil { + return fmt.Errorf("open snapshot %s: %w", source, err) + } + defer func() { retErr = errors.Join(retErr, in.Close()) }() + out, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, info.Mode().Perm()) + if err != nil { + return fmt.Errorf("create %s: %w", path, err) + } + defer func() { retErr = errors.Join(retErr, out.Close()) }() + if _, err := io.Copy(out, in); err != nil { + return fmt.Errorf("write %s: %w", path, err) + } + if err := out.Sync(); err != nil { + return fmt.Errorf("sync %s: %w", path, err) + } + return nil +} diff --git a/internal/store/maintenance_test.go b/internal/store/maintenance_test.go new file mode 100644 index 000000000..95eda0c86 --- /dev/null +++ b/internal/store/maintenance_test.go @@ -0,0 +1,92 @@ +package store + +import ( + "context" + "database/sql" + "errors" + "fmt" + "os" + "path/filepath" + "testing" +) + +// Ownership is what makes "the State Store is stopped" a fact an offline +// command can check, so the primitive itself is asserted here rather than only +// through the commands that use it: exclusivity, release, and per-database +// independence. +func TestOwnershipIsExclusivePerDatabase(t *testing.T) { + dir := t.TempDir() + first := filepath.Join(dir, "archie.db-tasks.sqlite") + second := filepath.Join(dir, "other.db-tasks.sqlite") + + held, err := AcquireOwnership(first) + if err != nil { + t.Fatalf("first AcquireOwnership: %v", err) + } + if _, err := AcquireOwnership(first); !errors.Is(err, ErrStoreOwned) { + t.Fatalf("second AcquireOwnership on the same database = %v, want ErrStoreOwned", err) + } + // A different database is a different claim: one process owning its store + // must not stop another from owning its own. + elsewhere, err := AcquireOwnership(second) + if err != nil { + t.Fatalf("AcquireOwnership on an unrelated database: %v", err) + } + if _, err := os.Stat(first + ".lock"); err != nil { + t.Fatalf("ownership left no sidecar to lock: %v", err) + } + + if err := held.Release(); err != nil { + t.Fatalf("Release: %v", err) + } + reacquired, err := AcquireOwnership(first) + if err != nil { + t.Fatalf("AcquireOwnership after Release: %v", err) + } + if err := errors.Join(reacquired.Release(), elsewhere.Release()); err != nil { + t.Fatalf("release on cleanup: %v", err) + } +} + +// The snapshot an update takes is the state a rollback restores, and it is +// restored by the release that wrote it. Verifying the copy must therefore not +// migrate it: a snapshot silently upgraded to this binary's schema would be +// refused by the older binary it exists to serve. +func TestBackupSnapshotKeepsTheSchemaVersionItCopied(t *testing.T) { + dir := t.TempDir() + database := filepath.Join(dir, "archie.db-tasks.sqlite") + snapshot := filepath.Join(dir, "snapshot.sqlite") + st, err := Open(t.Context(), database) + if err != nil { + t.Fatal(err) + } + if err := st.Close(); err != nil { + t.Fatal(err) + } + // An older release's file: the schema version it wrote, which this binary + // is expected to migrate when the store is next opened for serving. + if err := setUserVersion(t.Context(), database, taskSchemaVersion-1); err != nil { + t.Fatal(err) + } + + if err := Backup(t.Context(), database, snapshot); err != nil { + t.Fatalf("Backup: %v", err) + } + version, err := ValidateFile(t.Context(), snapshot) + if err != nil { + t.Fatalf("ValidateFile on the snapshot: %v", err) + } + if version != taskSchemaVersion-1 { + t.Fatalf("snapshot schema version = %d, want the %d it copied", version, taskSchemaVersion-1) + } +} + +func setUserVersion(ctx context.Context, path string, version int) error { + db, err := sql.Open("sqlite", path) + if err != nil { + return err + } + defer func() { _ = db.Close() }() + _, err = db.ExecContext(ctx, fmt.Sprintf(`PRAGMA user_version = %d`, version)) + return err +} diff --git a/internal/store/ownership.go b/internal/store/ownership.go new file mode 100644 index 000000000..b0743d89f --- /dev/null +++ b/internal/store/ownership.go @@ -0,0 +1,68 @@ +package store + +import ( + "errors" + "fmt" + "os" + "path/filepath" +) + +// ErrStoreOwned is returned when a task database is already owned by another +// process. Ownership is what makes "the State Store is stopped" a fact an +// offline command can check rather than a rule the operator is trusted to +// follow: overwriting a database a running server still holds leaves it +// writing to an unlinked inode while the store serves a file nobody owns. +var ErrStoreOwned = errors.New("task database is owned by another process") + +// Ownership is one process's exclusive claim over one task database, held for +// as long as that process owns the file. The serving State Store takes it for +// its whole life; the offline recovery commands take it for the duration of an +// operation that rewrites the file. +// +// The claim lives in a sidecar file beside the database rather than on the +// database inode itself, so it survives the rename a restore performs: after +// the swap the sidecar is still the one path every process locks. +type Ownership struct { + f *os.File +} + +// AcquireOwnership takes the store's ownership lock without blocking. A +// database another process owns comes back as ErrStoreOwned, never as a wait: +// a process that must not proceed has to hear so, and a blocking lock would +// turn a mistyped command into a hung terminal. +func AcquireOwnership(path string) (*Ownership, error) { + lockPath := ownershipLockPath(path) + if err := os.MkdirAll(filepath.Dir(lockPath), 0o755); err != nil { + return nil, fmt.Errorf("create store directory for %s: %w", lockPath, err) + } + f, err := os.OpenFile(lockPath, os.O_RDWR|os.O_CREATE, 0o644) + if err != nil { + return nil, fmt.Errorf("open store lock %s: %w", lockPath, err) + } + taken, err := lockFile(f) + if err != nil { + return nil, errors.Join(fmt.Errorf("lock %s: %w", lockPath, err), f.Close()) + } + if !taken { + return nil, errors.Join(fmt.Errorf("%s: %w", path, ErrStoreOwned), f.Close()) + } + return &Ownership{f: f}, nil +} + +// Release drops the claim and closes the sidecar. Closing the descriptor alone +// would release it too; the explicit unlock keeps the intent readable and +// makes Release idempotent. +func (o *Ownership) Release() error { + if o == nil || o.f == nil { + return nil + } + err := unlockFile(o.f) + f := o.f + o.f = nil + return errors.Join(err, f.Close()) +} + +// ownershipLockPath is the sidecar every owner of path locks. It is derived +// from the database path, so the State Store process and an offline command +// that were given the same database agree on it without configuration. +func ownershipLockPath(path string) string { return path + ".lock" } diff --git a/internal/store/ownership_unix.go b/internal/store/ownership_unix.go new file mode 100644 index 000000000..3d0d2bb4b --- /dev/null +++ b/internal/store/ownership_unix.go @@ -0,0 +1,30 @@ +//go:build unix + +package store + +import ( + "errors" + "os" + "syscall" +) + +// lockFile takes a non-blocking exclusive flock on f. The boolean reports +// whether it was taken: a lock another process holds (EWOULDBLOCK) is a normal +// answer to "is the State Store running?", not an error. +func lockFile(f *os.File) (bool, error) { + err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) + if errors.Is(err, syscall.EWOULDBLOCK) { + return false, nil + } + if err != nil { + return false, err + } + return true, nil +} + +func unlockFile(f *os.File) error { + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_UN); err != nil { + return err + } + return nil +} diff --git a/internal/store/ownership_windows.go b/internal/store/ownership_windows.go new file mode 100644 index 000000000..e282ae29d --- /dev/null +++ b/internal/store/ownership_windows.go @@ -0,0 +1,23 @@ +//go:build windows + +// flock is a POSIX primitive and Windows is not a supported deployment surface +// (CLAUDE.md's deployment model is Linux and macOS), so this file fails closed: +// every lock attempt reports an error rather than silently degrading to "no +// lock". An offline command that cannot prove it owns the database must refuse +// to rewrite it, which is the only safe reading of an unavailable primitive. +// +// If a Windows deployment ever becomes supported, swap this for a real +// LockFileEx/UnlockFileEx pair; nothing else in the package needs to change. + +package store + +import ( + "errors" + "os" +) + +func lockFile(*os.File) (bool, error) { + return false, errors.New("store ownership locking requires a POSIX flock, which this platform does not provide") +} + +func unlockFile(*os.File) error { return nil } diff --git a/scripts/archie-update-install b/scripts/archie-update-install index ce3635f63..30c43bef4 100755 --- a/scripts/archie-update-install +++ b/scripts/archie-update-install @@ -183,10 +183,18 @@ fi # that cannot take one must not start: once binaries are replaced, a failure # leaves the previous release unable to read the store it just migrated. This # runs ahead of the adapter refresh below so a refusal changes nothing at all. +# +# The snapshot is taken by the recovery subcommand of the release being +# installed, from its own checkout, rather than by hand-rolled sqlite3 shell: +# the operator running `archie-state-store backup` by hand and the installer +# are then the same implementation, and the two cannot drift. The installed +# binary is the previous release, which may predate the subcommand, so it +# cannot be the one asked -- and a release whose recovery path will not build is a +# release that cannot back up, which is exactly what this pre-flight refuses. if [[ ",$components," == *,daemon,* ]] && [ -n "$TASK_DB_PATH" ] && [ -f "$TASK_DB_PATH" ]; then echo "==> backing up task database for schema-safe rollback" mkdir -p "$(dirname "$TASK_DB_BACKUP")" - if ! sqlite3 "$TASK_DB_PATH" ".backup '$TASK_DB_BACKUP'"; then + if ! go run ./cmd/archie-state-store backup -db "$TASK_DB_PATH" -out "$TASK_DB_BACKUP"; then echo "refusing to install $GW: could not back up the task database at $TASK_DB_PATH, so a failed update could not be rolled back" >&2 exit 1 fi From 8d015db7c15198a85d5810f32495fdea1f46e17e Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Mon, 21 Sep 2026 20:33:59 +1000 Subject: [PATCH 2/4] fix(statestore): make validate run boot's gate, and stop backup replacing the database Two P1 findings from review, both about a claim outrunning the code. validate answered a different question from boot's. Boot refuses on configuration.Validate over the file config with every stored resource layered onto it (internal/app/archied/control_plane.go:27), which is a stricter check than the write path's per-resource Decode: validateScheduling never inspects dispatch.trigger and explicitly permits "0s", while boot rejects an unknown trigger and a non-positive poll interval. A store holding such a value printed "is a valid store ... N stored resources validate" and exited 0 while archied exited 1 on it -- a false green pushing an operator toward the blunt restore instead of the surgical rollback. validate now runs both layers and reports both, in boot's order: store.ValidateFile, then the write path's ValidateStored, then the config gate. The layering itself is now one implementation: controlplane's runtimeConfigFrom takes a resourceReader, which the gRPC client and a new Server.StoredRuntimeConfig (reading its own store) both provide, so the offline check cannot drift from the daemon's. validate takes -config / -config-overlay (default: the path the serve subcommand uses), because the question is about the stored values layered onto that document. backup had no guard against -out naming the database. It renames its snapshot over the target, so `backup -db F -out F` replaced a database a serving State Store may still have open and orphaned the WAL of the file it just unlinked -- the state Restore refuses up front and maintenance.go's own comment calls unrecoverable. backup now refuses with the same message Restore uses, which is what makes its exemption from the ownership lock sound. Claims corrected where they ran ahead of the code: ValidateStored now says it is the write path's check and not boot's, the Restore comment no longer promises "never a mixture" for the sidecars it removes first, the usage text advertises validate's -config, and the architecture table drops "must be stopped" for validate (nothing enforces it; read-only opens skip the lock by design) and states which two layers validate reports. Tests: TestRecoveryValidateRunsTheGateBootRuns fails on the pre-fix code ("validate accepted a stored policy the daemon refuses to boot with") and covers a store the State Store never seeded; TestRecoveryBackupRefuses- TheDatabaseAsItsOwnSnapshot fails on the pre-fix code ("backup over the database itself succeeded"). The validate-accept test now seeds the store through ImportConfig and asserts the reported count is non-zero, and the stored-resource case uses a value both gates refuse instead of one only the write path does. --- cmd/archie-state-store/recovery.go | 22 ++- cmd/archie-state-store/recovery_test.go | 128 ++++++++++++++++-- docs/architecture/safe-change-and-recovery.md | 26 ++-- internal/app/archied/state_store_recovery.go | 76 +++++++++-- internal/app/controlplane/registry.go | 11 +- internal/app/controlplane/runtime_config.go | 83 +++++++++--- internal/store/maintenance.go | 18 ++- 7 files changed, 310 insertions(+), 54 deletions(-) diff --git a/cmd/archie-state-store/recovery.go b/cmd/archie-state-store/recovery.go index 00ee75750..4b537e9ff 100644 --- a/cmd/archie-state-store/recovery.go +++ b/cmd/archie-state-store/recovery.go @@ -5,10 +5,23 @@ import ( "flag" "fmt" "io" + "os" + "path/filepath" "github.com/samcharles93/archie-core/internal/app/archied" ) +// defaultConfigPath is the configuration the serve path reads by default; the +// offline validate asks what the daemon would boot with, so it reads the same +// one. +func defaultConfigPath() string { + base, err := os.UserConfigDir() + if err != nil { + return "" + } + return filepath.Join(base, "archie", "config.toml") +} + // recoveryUsage is the whole flag surface of the offline recovery commands. // They are subcommands of this binary rather than a recovery binary of their // own: offline database maintenance belongs to the process that owns the file @@ -25,7 +38,8 @@ owns it, while backup takes a consistent snapshot of a serving store. backup -db FILE -out FILE write a snapshot of the database restore -db FILE -from FILE replace the database with a snapshot - validate -db FILE check the database the way boot does + validate -db FILE [-config FILE] check the file, its stored settings and the + configuration the daemon would boot with rollback -db FILE -kind KIND replay an earlier revision of a stored [-revision N] resource through the ordinary replace ` @@ -56,6 +70,12 @@ func runRecovery(args []string, stdout, stderr io.Writer) int { flags.StringVar(&options.From, "from", "", "snapshot file restore reads") flags.StringVar(&options.Kind, "kind", "", "control-plane resource kind rollback replays") flags.Int64Var(&options.Revision, "revision", 0, "revision rollback replays (default: the newest one older than the current value)") + // Only validate asks a question about the process rather than the file, so + // only validate takes the configuration the process would boot with. + if command == archied.RecoveryValidate { + flags.StringVar(&options.Config, "config", defaultConfigPath(), "configuration file or directory the daemon boots with") + flags.StringVar(&options.Overlay, "config-overlay", "", "configuration overlay file or directory") + } if err := flags.Parse(args[1:]); err != nil { return 2 } diff --git a/cmd/archie-state-store/recovery_test.go b/cmd/archie-state-store/recovery_test.go index efe96c4f9..187b89762 100644 --- a/cmd/archie-state-store/recovery_test.go +++ b/cmd/archie-state-store/recovery_test.go @@ -18,6 +18,7 @@ import ( "fmt" "os" "path/filepath" + "strconv" "strings" "testing" @@ -25,6 +26,7 @@ import ( "github.com/samcharles93/archie-core/internal/app/archied" "github.com/samcharles93/archie-core/internal/app/controlplane" + "github.com/samcharles93/archie-core/internal/infrastructure/configuration" "github.com/samcharles93/archie-core/internal/store" ) @@ -142,6 +144,28 @@ func TestRecoveryBackupRefusesAMissingDatabase(t *testing.T) { } } +// backup is the one recovery command allowed to run against a serving store, +// and that exemption is only sound while it never replaces the database. With +// -out naming the database it renames the snapshot over the live file and +// leaves the WAL of the database it just unlinked -- the state the restore path +// refuses up front and the code elsewhere calls unrecoverable. +func TestRecoveryBackupRefusesTheDatabaseAsItsOwnSnapshot(t *testing.T) { + db := filepath.Join(t.TempDir(), "archie.db-tasks.sqlite") + st := openTaskStore(t, db) + seedTask(t, st, "live") + + code, _, stderr := runRecoveryCmd(t, "backup", "-db", db, "-out", db) + if code == 0 { + t.Fatal("backup over the database itself succeeded") + } + if !strings.Contains(stderr, "is the database itself") { + t.Errorf("refusal must say the snapshot is the database; stderr = %q", stderr) + } + if got := taskTitles(t, db); len(got) != 1 || got[0] != "live" { + t.Fatalf("refused backup changed the database: tasks = %v", got) + } +} + // Restoring is the forward-migration escape hatch: the previous release // reads this file, so it must be exactly the snapshot, with the WAL that // belongs to the replaced database gone rather than replayed over it. @@ -242,18 +266,68 @@ func TestRecoveryRestoreRefusesTheDatabaseAsItsOwnSnapshot(t *testing.T) { func TestRecoveryValidateAcceptsAHealthyStore(t *testing.T) { dir := t.TempDir() + configPath := writeMinimalConfig(t, dir) db := filepath.Join(dir, "archie.db-tasks.sqlite") st := openTaskStore(t, db) seedTask(t, st, "healthy") - seedResource(t, st, controlplane.ModelRoleAssignmentsKind, `{"implement":"anthropic/claude"}`, "seed") + seedStoreResources(t, st, configPath) - code, stdout, stderr := runRecoveryCmd(t, "validate", "-db", db) + code, stdout, stderr := runRecoveryCmd(t, "validate", "-db", db, "-config", configPath) if code != 0 { t.Fatalf("validate exited %d: %s", code, stderr) } if !strings.Contains(stdout, "schema version") { t.Errorf("validate must report the schema version it verified; stdout = %q", stdout) } + // The count is part of what the operator reads, and a check that reports + // "0 stored resources" for a store full of them is worse than no check. + if count := storedResourceCount(t, stdout); count == 0 { + t.Errorf("validate reported no stored resources for a seeded store; stdout = %q", stdout) + } +} + +// validate's verdict is only useful if it is the daemon's verdict. Boot refuses +// on configuration.Validate over the file config with every stored resource +// layered onto it (internal/app/archied/control_plane.go), not on the write +// path's own per-resource decode, and the two disagree: the write path accepts +// a scheduling policy boot rejects. +func TestRecoveryValidateRunsTheGateBootRuns(t *testing.T) { + dir := t.TempDir() + configPath := writeMinimalConfig(t, dir) + db := filepath.Join(dir, "archie.db-tasks.sqlite") + seedStoreResources(t, openTaskStore(t, db), configPath) + + // The control: a store seeded from this config is one the daemon boots on. + code, _, stderr := runRecoveryCmd(t, "validate", "-db", db, "-config", configPath) + if code != 0 { + t.Fatalf("validate exited %d on a store the daemon starts on: %s", code, stderr) + } + + // A hand-edited value, or an older revision of one: the write path's own + // validator never inspects dispatch.trigger, so only the config gate + // catches it, and the daemon exits 1 on it (validateDispatch). + if err := execRaw(t, db, `UPDATE resources SET value='{"poll_interval":"1m","max_retries":3,"dispatch":{"trigger":"bogus"}}' WHERE kind='scheduling-policy'`); err != nil { + t.Fatal(err) + } + code, stdout, stderr := runRecoveryCmd(t, "validate", "-db", db, "-config", configPath) + if code == 0 { + t.Fatalf("validate accepted a stored policy the daemon refuses to boot with; stdout = %q", stdout) + } + if !strings.Contains(stderr, "dispatch.trigger") { + t.Errorf("refusal must name what boot rejects; stderr = %q", stderr) + } + + // A store the State Store never seeded: the daemon cannot layer its + // settings from it either, so the offline check must not call it valid. + unseeded := filepath.Join(dir, "unseeded.db-tasks.sqlite") + openTaskStore(t, unseeded) + code, stdout, stderr = runRecoveryCmd(t, "validate", "-db", unseeded, "-config", configPath) + if code == 0 { + t.Fatalf("validate accepted a store with no stored settings; stdout = %q", stdout) + } + if !strings.Contains(stderr, "provider-settings") { + t.Errorf("refusal must name the resource kind boot cannot read; stderr = %q", stderr) + } } // validate exists to answer "would archied start against this file". Each case @@ -318,16 +392,19 @@ func TestRecoveryValidateRefusesWhatTheDaemonRefuses(t *testing.T) { }) t.Run("stored_resource_that_will_not_validate", func(t *testing.T) { - // The stored settings value that stops archied starting: it can only - // get here by bypassing the write path, which is exactly the state the - // control plane fails closed on and the operator has no path back from. + // A stored value that both gates refuse: the write path stops it at + // Decode (workflow.ExecutionSettings.Validate rejects a negative limit), + // and boot stops on it too -- startWorkflowExecutionSettings decodes the + // same document and exits 1. It can only get here by bypassing the write + // path, which is exactly the state the operator has no path back from. db := newStore(t) - seedResource(t, openTaskStore(t, db), controlplane.ModelRoleAssignmentsKind, `{"implement":"claude"}`, "hand-edited") + seedResource(t, openTaskStore(t, db), controlplane.WorkflowExecutionSettingsKind, + `{"max_model_tool_steps":-1,"max_runtime_seconds":60,"max_consecutive_gate_failures":3}`, "hand-edited") code, _, stderr := runRecoveryCmd(t, "validate", "-db", db) if code == 0 { t.Fatal("validate accepted a stored resource the write path would refuse") } - if !strings.Contains(stderr, controlplane.ModelRoleAssignmentsKind) { + if !strings.Contains(stderr, controlplane.WorkflowExecutionSettingsKind) { t.Errorf("refusal must name the resource kind; stderr = %q", stderr) } }) @@ -351,11 +428,14 @@ func TestRecoveryValidateRefusesWhatTheDaemonRefuses(t *testing.T) { // settings. The serving process creates the table on its next start, so // refusing this file would send an operator to restore a snapshot for a // database that would have started perfectly well. - db := newStore(t) + dir := t.TempDir() + configPath := writeMinimalConfig(t, dir) + db := filepath.Join(dir, "archie.db-tasks.sqlite") + openTaskStore(t, db) if err := execRaw(t, db, `DROP TABLE resources`); err != nil { t.Fatal(err) } - code, stdout, stderr := runRecoveryCmd(t, "validate", "-db", db) + code, stdout, stderr := runRecoveryCmd(t, "validate", "-db", db, "-config", configPath) if code != 0 { t.Fatalf("validate exited %d: %s", code, stderr) } @@ -502,6 +582,36 @@ func TestRecoveryRejectsAnUnknownSubcommand(t *testing.T) { } } +// seedStoreResources seeds the store the way the State Store seeds it at boot, +// from the same config file the operator hands validate: the command's verdict +// is about the pair, and a store whose settings never matched the config is a +// state boot does not recognise either. +func seedStoreResources(t *testing.T, st *store.Store, configPath string) { + t.Helper() + doc, err := configuration.New(nil).Resolve(configPath, "") + if err != nil { + t.Fatalf("resolve %s: %v", configPath, err) + } + if _, err := controlplane.NewServer(st).ImportConfig(t.Context(), doc.Config); err != nil { + t.Fatalf("seed store resources: %v", err) + } +} + +// storedResourceCount reads the count validate reports out of its summary line. +func storedResourceCount(t *testing.T, stdout string) int { + t.Helper() + before, found := strings.CutSuffix(stdout, " stored resources validate\n") + if !found { + t.Fatalf("no resource count in %q", stdout) + } + fields := strings.Fields(before) + count, err := strconv.Atoi(fields[len(fields)-1]) + if err != nil { + t.Fatalf("resource count in %q: %v", stdout, err) + } + return count +} + // seedResource writes one revision of a control-plane resource through the // store, the way the control plane's replace does. The request ID is derived // from the label and the revision it produces: the store deduplicates a diff --git a/docs/architecture/safe-change-and-recovery.md b/docs/architecture/safe-change-and-recovery.md index 6f10f70ff..d9b409f0b 100644 --- a/docs/architecture/safe-change-and-recovery.md +++ b/docs/architecture/safe-change-and-recovery.md @@ -127,15 +127,23 @@ Nothing skips the control plane at boot, and the only writer for a resource is the daemon fails closed without. `archie-state-store` therefore carries the offline recovery commands (`archie-core-6zw0`). They work on the task database file directly -- the configured `db_path` with `-tasks.sqlite` appended, not -`db_path` itself -- and need neither `archied` nor the Web UI. +`db_path` itself -- and need neither `archied` nor the Web UI running: `validate` +reads the configuration file the daemon boots with, which is what makes its +verdict the daemon's verdict. | Command | What it does | State Store | | --- | --- | --- | | `backup -db F -out F` | writes a transactionally consistent snapshot (`VACUUM INTO`), verified as a usable store before it is published | may be running | | `restore -db F -from F` | replaces the database with a snapshot and drops the replaced file's `-wal`/`-shm` | must be stopped | -| `validate -db F` | checks the file the way boot meets it: readable, not corrupt, schema no newer than the binary, and every stored resource decoding under the definition that owns it | must be stopped | +| `validate -db F [-config C]` | runs the checks that stop archied starting: the file is readable, uncorrupted and no newer than the binary; every stored resource decodes under the definition that owns it; and `configuration.Validate` accepts the stored values layered onto `C`, the check the daemon reports as `validate database settings` | may be running | | `rollback -db F -kind K [-revision N]` | replays an earlier revision of a stored resource through the ordinary replace, recording the rollback as a new revision | must be stopped | +The two validation layers `validate` reports are not the same check, and it +reports both rather than merging them: the write path's own decode is stricter +about shape (it rejects unknown fields boot's decode ignores), while boot is +stricter about meaning (only boot checks `dispatch.trigger`, and a positive +`poll_interval`). Either one refusing is a store somebody has to look at. + No rollback RPC exists and none is needed: a revision carries its own value, so restoring one is the same replace the dashboard and the channels already use. The offline command is that operation with the store stopped -- the one state in @@ -144,12 +152,14 @@ which the daemon's own API cannot serve it. "Must be stopped" is checked rather than assumed. The serving process takes an exclusive flock on `.lock` for its whole life, and `restore` and `rollback` refuse with `task database is owned by another process` while another process -holds it. `backup` is the deliberate exception: the update installer runs inside -the process it is replacing and cannot stop the State Store, so the snapshot -that makes a failed update reversible is taken against a serving store -- and -`scripts/archie-update-install` takes it through this subcommand rather than -through its own `sqlite3` shell, so the installer and an operator run one -implementation. +holds it. The read-only commands -- `backup` and `validate` -- take no lock and +are meant to run against a serving store: `backup` because the update installer +runs inside the process it is replacing and cannot stop the State Store, and +`validate` because an operator checks a running deployment with it. That +`backup` exemption is only sound while backup never replaces the database, so +naming the database as its own `-out` is refused. `scripts/archie-update-install` +takes its pre-flight snapshot through this subcommand rather than through its +own `sqlite3` shell, so the installer and an operator run one implementation. `rollback` is the surgical route back from a bad stored value; `restore` is the blunt one, and is also the way back from a schema migration a release cannot diff --git a/internal/app/archied/state_store_recovery.go b/internal/app/archied/state_store_recovery.go index bf0acc93d..868755366 100644 --- a/internal/app/archied/state_store_recovery.go +++ b/internal/app/archied/state_store_recovery.go @@ -20,7 +20,9 @@ import ( "time" "github.com/samcharles93/archie-core/internal/app/controlplane" + "github.com/samcharles93/archie-core/internal/config" pb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1" + "github.com/samcharles93/archie-core/internal/infrastructure/configuration" "github.com/samcharles93/archie-core/internal/store" ) @@ -53,6 +55,12 @@ type StateStoreRecoveryOptions struct { Out string // From is the snapshot restore reads. From string + // Config and Overlay are the configuration the daemon would boot with. + // validate asks whether the stored settings are a state archied starts on, + // and that question is about the stored values *layered onto this + // configuration*, so the file config is part of the answer. + Config string + Overlay string // Kind is the control-plane resource kind rollback replays. Kind string // Revision is the revision rollback replays. Zero means the newest @@ -84,7 +92,7 @@ func RunStateStoreRecovery(ctx context.Context, options StateStoreRecoveryOption if options.DB == "" { return "", errors.New("validate requires -db") } - return validateStore(ctx, options.DB) + return validateStore(ctx, options) case RecoveryRollback: if options.DB == "" || options.Kind == "" { return "", errors.New("rollback requires -db and -kind") @@ -95,20 +103,35 @@ func RunStateStoreRecovery(ctx context.Context, options StateStoreRecoveryOption } } -// validateStore answers "would archied start against this file". The file-level -// checks come first, so the store is only read once its schema is one this -// binary understands; the resource check then decodes every stored value with -// the definition that owns it, which is the same validation a write applies. -func validateStore(ctx context.Context, path string) (string, error) { - version, err := store.ValidateFile(ctx, path) +// validateStore answers the question an operator actually has: would archied +// start against this file. It runs the checks that stop it, in the order boot +// meets them. +// +// First the file itself, since a database that is corrupt, newer than this +// binary, or not a database is one the serving process refuses to open. Then +// the writer's own per-resource validation, which is what a hand-edited value +// violated. Then boot's gate: the file config with every stored resource +// layered onto it, and configuration.Validate -- the check whose failure the +// daemon reports as "validate database settings". +// +// The two validation layers disagree in both directions, and both are reported +// rather than merged. The write path is stricter about shape (it rejects +// unknown fields boot's decode ignores), while boot is stricter about meaning +// (only boot checks dispatch.trigger and a positive poll interval, both of +// which the write path's own validators let through). A store either check +// refuses is a store somebody has to look at. +func validateStore(ctx context.Context, options StateStoreRecoveryOptions) (string, error) { + version, err := store.ValidateFile(ctx, options.DB) if err != nil { return "", err } - st, err := store.OpenReadOnly(ctx, path) + st, err := store.OpenReadOnly(ctx, options.DB) if err != nil { return "", err } defer func() { _ = st.Close() }() + + server := controlplane.NewServer(st) // The resources table is only absent from a store written before the control // plane existed. That store holds no settings to validate, and the serving // process creates the table on its next start -- refusing it here would send @@ -119,12 +142,43 @@ func validateStore(ctx context.Context, path string) (string, error) { return "", err } if stored { - checked, err = controlplane.NewServer(st).ValidateStored(ctx) - if err != nil { + if checked, err = server.ValidateStored(ctx); err != nil { return "", err } } - return fmt.Sprintf("%s is a valid store at schema version %d; %d stored resources validate", path, version, checked), nil + + base, err := bootConfig(ctx, options) + if err != nil { + return "", err + } + document := base + if stored { + // A store the State Store never seeded has no kinds to layer, and the + // daemon cannot start against it either: the State Store seeds every kind + // from its own config at boot. + if document, _, err = server.StoredRuntimeConfig(ctx, base); err != nil { + return "", err + } + } + if err := configuration.Validate(&document); err != nil { + // The daemon's own wording, so an operator can match this failure to the + // one that stopped archied starting. + return "", fmt.Errorf("validate database settings: %w", err) + } + return fmt.Sprintf("%s is a valid store at schema version %d; %d stored resources validate", options.DB, version, checked), nil +} + +// bootConfig resolves the configuration the daemon would boot with. validate is +// the only recovery command that asks a question about the process rather than +// the file, and that question is about the stored values layered onto this +// document, so the document is part of the answer. +func bootConfig(ctx context.Context, options StateStoreRecoveryOptions) (config.Config, error) { + b := newBootstrap() + defer b.cleanup() + if err := b.loadConfig(ctx, options.Config, options.Overlay); err != nil { + return config.Config{}, err + } + return b.cfg, nil } // rollbackResource is the one operation the Web UI cannot be asked to perform diff --git a/internal/app/controlplane/registry.go b/internal/app/controlplane/registry.go index ddf66142a..bbfa206e5 100644 --- a/internal/app/controlplane/registry.go +++ b/internal/app/controlplane/registry.go @@ -102,10 +102,13 @@ func (s *Server) Owns(kind string) bool { // resources it checked. Errors name the kind and the revision it was stored at, // which is what an operator needs to roll the value back. // -// This is how an offline validate reproduces the refusal that stops archied -// starting: the check is the writer's own, so it cannot drift from what a -// replace would accept, and a value can only be stored unvalidated by bypassing -// the write path entirely. +// It is the write path's own check, not boot's: boot refuses on +// configuration.Validate over the stored values layered onto the file config, +// and the two disagree in both directions (this one rejects unknown fields +// boot's decode ignores, while only boot checks dispatch.trigger and a positive +// poll interval). A caller answering "would the daemon start" therefore needs +// StoredRuntimeConfig and configuration.Validate as well -- see +// validateStore in internal/app/archied/state_store_recovery.go. func (s *Server) ValidateStored(ctx context.Context) (int, error) { checked := 0 var failures []error diff --git a/internal/app/controlplane/runtime_config.go b/internal/app/controlplane/runtime_config.go index c98f0d86e..93fb6f9d2 100644 --- a/internal/app/controlplane/runtime_config.go +++ b/internal/app/controlplane/runtime_config.go @@ -10,6 +10,15 @@ import ( pb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1" ) +// resourceReader is the one read the layering performs: a resource kind's value +// and the version it came from. The gRPC client and the in-process server both +// provide it, so "what the database covers in the running config" has exactly +// one implementation. The offline validate runs the same code as boot, which is +// the only way its verdict can be boot's verdict. +type resourceReader interface { + query(ctx context.Context, kind string, decode func([]byte) error) (int64, error) +} + // RuntimeConfig applies restart-scoped database resources over bootstrap // configuration. Values intentionally absent from a control-plane projection, // such as secret-bearing MCP headers and channel update commands, remain @@ -19,9 +28,47 @@ import ( // layers them in can report which version it is running // (docs/prds/control-plane-apply-status.md). func (c *Client) RuntimeConfig(ctx context.Context, base config.Config) (config.Config, map[string]int64, error) { + return runtimeConfigFrom(ctx, c, base) +} + +// RuntimeChatConfig layers the stored channel settings over the file document's +// chat section. +func (c *Client) RuntimeChatConfig(ctx context.Context, base config.ChatConfig) (config.ChatConfig, int64, error) { + return runtimeChatConfigFrom(ctx, c, base) +} + +// StoredRuntimeConfig is RuntimeConfig's layering over the store this server +// owns, for a caller holding the database file rather than a connection to it. +// Boot layers the stored settings onto the file config and then runs +// configuration.Validate, so an offline check of that database has to make the +// same first move with the same implementation; called with the config the +// daemon would load, the result is the document boot decides on. +func (s *Server) StoredRuntimeConfig(ctx context.Context, base config.Config) (config.Config, map[string]int64, error) { + return runtimeConfigFrom(ctx, storeReader{resources: s.store}, base) +} + +// storeReader reads the server's own store. It mirrors what the gRPC read +// answers -- a missing kind is an error, not an empty value -- so a store the +// daemon cannot layer settings from is one this reports rather than skips. +type storeReader struct { + resources ResourceStore +} + +func (r storeReader) query(ctx context.Context, kind string, decode func([]byte) error) (int64, error) { + resource, err := r.resources.Resource(ctx, kind) + if err != nil { + return 0, fmt.Errorf("read %s: %w", kind, err) + } + if err := decode(resource.Value); err != nil { + return 0, fmt.Errorf("decode %s: %w", kind, err) + } + return resource.Version, nil +} + +func runtimeConfigFrom(ctx context.Context, reader resourceReader, base config.Config) (config.Config, map[string]int64, error) { out := base.Clone() versions := map[string]int64{} - if err := c.queryInto(ctx, versions, ProviderSettingsKind, func(value []byte) error { + if err := layerResource(ctx, reader, versions, ProviderSettingsKind, func(value []byte) error { var providers map[string]providerDocument if err := json.Unmarshal(value, &providers); err != nil { return err @@ -34,18 +81,18 @@ func (c *Client) RuntimeConfig(ctx context.Context, base config.Config) (config. }); err != nil { return config.Config{}, nil, err } - if err := c.queryJSONInto(ctx, versions, ModelRoleAssignmentsKind, &out.Models); err != nil { + if err := layerResourceJSON(ctx, reader, versions, ModelRoleAssignmentsKind, &out.Models); err != nil { return config.Config{}, nil, err } - if err := c.queryJSONInto(ctx, versions, RepositoryPoliciesKind, &out.Repos); err != nil { + if err := layerResourceJSON(ctx, reader, versions, RepositoryPoliciesKind, &out.Repos); err != nil { return config.Config{}, nil, err } - chat, chatVersion, err := c.RuntimeChatConfig(ctx, out.Chat) + chat, chatVersion, err := runtimeChatConfigFrom(ctx, reader, out.Chat) if err != nil { return config.Config{}, nil, err } out.Chat, versions[ChannelSettingsKind] = chat, chatVersion - if err := c.queryInto(ctx, versions, SchedulingPolicyKind, func(value []byte) error { + if err := layerResource(ctx, reader, versions, SchedulingPolicyKind, func(value []byte) error { var policy schedulingPolicy if err := json.Unmarshal(value, &policy); err != nil { return err @@ -59,11 +106,11 @@ func (c *Client) RuntimeConfig(ctx context.Context, base config.Config) (config. }); err != nil { return config.Config{}, nil, err } - return c.runtimeToolConfig(ctx, versions, out) + return runtimeToolConfigFrom(ctx, reader, versions, out) } -func (c *Client) runtimeToolConfig(ctx context.Context, versions map[string]int64, out config.Config) (config.Config, map[string]int64, error) { - if err := c.queryInto(ctx, versions, ToolSettingsKind, func(value []byte) error { +func runtimeToolConfigFrom(ctx context.Context, reader resourceReader, versions map[string]int64, out config.Config) (config.Config, map[string]int64, error) { + if err := layerResource(ctx, reader, versions, ToolSettingsKind, func(value []byte) error { var settings toolSettings if err := json.Unmarshal(value, &settings); err != nil { return err @@ -81,7 +128,7 @@ func (c *Client) runtimeToolConfig(ctx context.Context, versions map[string]int6 }); err != nil { return config.Config{}, nil, err } - if err := c.queryInto(ctx, versions, PluginSettingsKind, func(value []byte) error { + if err := layerResource(ctx, reader, versions, PluginSettingsKind, func(value []byte) error { var settings pluginSettings if err := json.Unmarshal(value, &settings); err != nil { return err @@ -91,15 +138,15 @@ func (c *Client) runtimeToolConfig(ctx context.Context, versions map[string]int6 }); err != nil { return config.Config{}, nil, err } - if err := c.queryJSONInto(ctx, versions, ContainerRuntimePoliciesKind, &out.Containers); err != nil { + if err := layerResourceJSON(ctx, reader, versions, ContainerRuntimePoliciesKind, &out.Containers); err != nil { return config.Config{}, nil, err } return out, versions, nil } -func (c *Client) RuntimeChatConfig(ctx context.Context, base config.ChatConfig) (config.ChatConfig, int64, error) { +func runtimeChatConfigFrom(ctx context.Context, reader resourceReader, base config.ChatConfig) (config.ChatConfig, int64, error) { out := base - version, err := c.query(ctx, ChannelSettingsKind, func(value []byte) error { + version, err := reader.query(ctx, ChannelSettingsKind, func(value []byte) error { var settings channelSettings if err := json.Unmarshal(value, &settings); err != nil { return err @@ -117,10 +164,10 @@ func (c *Client) RuntimeChatConfig(ctx context.Context, base config.ChatConfig) return out, version, err } -// queryInto decodes a resource and records the version it came from, so -// RuntimeConfig ends up holding the version of every kind it layered in. -func (c *Client) queryInto(ctx context.Context, versions map[string]int64, kind string, decode func([]byte) error) error { - version, err := c.query(ctx, kind, decode) +// layerResource decodes a resource and records the version it came from, so the +// layering ends up holding the version of every kind it applied. +func layerResource(ctx context.Context, reader resourceReader, versions map[string]int64, kind string, decode func([]byte) error) error { + version, err := reader.query(ctx, kind, decode) if err != nil { return err } @@ -128,8 +175,8 @@ func (c *Client) queryInto(ctx context.Context, versions map[string]int64, kind return nil } -func (c *Client) queryJSONInto(ctx context.Context, versions map[string]int64, kind string, target any) error { - return c.queryInto(ctx, versions, kind, func(value []byte) error { return json.Unmarshal(value, target) }) +func layerResourceJSON(ctx context.Context, reader resourceReader, versions map[string]int64, kind string, target any) error { + return layerResource(ctx, reader, versions, kind, func(value []byte) error { return json.Unmarshal(value, target) }) } // query returns the version of the resource it decoded, so callers that layer diff --git a/internal/store/maintenance.go b/internal/store/maintenance.go index 12c3af029..d1165a600 100644 --- a/internal/store/maintenance.go +++ b/internal/store/maintenance.go @@ -133,6 +133,16 @@ func Backup(ctx context.Context, database, snapshot string) error { if err := RequireDatabase(database); err != nil { return err } + // The exemption from the ownership lock below is only sound while backup + // never replaces the database. With -out naming the database it would + // rename the snapshot over a file a serving State Store may still have + // open, orphaning the WAL of the file it just unlinked -- so it is refused + // exactly as Restore refuses it. + if same, err := sameFile(database, snapshot); err != nil { + return err + } else if same { + return fmt.Errorf("snapshot %s is the database itself", snapshot) + } if err := os.MkdirAll(filepath.Dir(snapshot), 0o755); err != nil { return fmt.Errorf("create snapshot directory: %w", err) } @@ -163,9 +173,11 @@ func Backup(ctx context.Context, database, snapshot string) error { // Restore replaces database with the snapshot at snapshot. The snapshot is // verified before anything is destroyed, so a file that is not a usable store -// cannot take the database down with it, and the database is replaced by a -// rename, so a failure leaves either the old file or the new one rather than a -// mixture. +// cannot take the database down with it, and the database file itself is +// replaced by a rename, so a failure leaves either the old file or the new one +// rather than a mixture. The replaced file's -wal/-shm are removed first, which +// the comment at that step explains: between the two calls the database is +// merely incomplete, and re-running the command completes it. // // It holds the ownership lock for its duration: rewriting a database a running // State Store still has open would leave that process writing to an unlinked From b5b58c370ae151afad5af54264ff65ea3c4d05c3 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Mon, 21 Sep 2026 20:40:47 +1000 Subject: [PATCH 3/4] fix(statestore): read a kind the store does not hold as its seed, and stop validate touching the daemon's log Four defects from the re-run contract review, all in the recovery path. A store missing a control-plane kind was refused while a store with no resources table at all was accepted -- two stances inside validateStore for states the daemon cannot tell apart. The State Store seeds every kind from its own config before it serves (RunStateStore -> ImportConfig), so a kind the file does not hold yet is one archied reads the seed for: an upgrade that adds a kind, and a validate before the State Store has started again, exited 1 with "read provider-settings: resource not found" for a store the daemon boots on fine. StoredRuntimeConfig now reads a missing kind as the seed the State Store would write for it (Definition.seededValue, shared with ImportConfig so the two cannot drift), and the missing-table branch is the same stance taken to its extreme. defaultConfigPath() derived the daemon's config with os.UserConfigDir, which disagrees with archied's configHome on darwin and rejects a relative XDG_CONFIG_HOME outright -- leaving validate with an empty path and no verdict, on a host where archied boots from relative-dir/archie/config.toml. It is now the daemon's own rule, exported as archied.DefaultConfigPath and used by the daemon's own -config default too, with a test pinning the equality. -h exited 2, the code documented as "the command line was wrong", while the serve path in the same binary exits 0 and main.go advertises -h for these subcommands. ErrHelp now returns 0. validate resolved the boot config through loadConfig -> setupLogging, which created cfg.Log.File's directory, opened the log, and wrote a line stamped component="daemon" -- a diagnostic editing the deployment it inspects, and rotating a log the daemon is writing. boot.stderrLog keeps setupLogging on the stderr logger for the recovery path: no feed, no task-log registry, no durable copy. Tests: TestRecoveryValidateRunsTheGateBootRuns now asserts the unseeded and the missing-kind stores are accepted (both exited 1 before), while the bogus-policy case still refuses; TestRecoveryHelpExitsZero, TestRecoveryValidateLeavesTheDaemonLogAlone and TestRecoveryDefaultConfigFollowsTheDaemonsConfigHome are new. Each was run red first, and the seed fallback and stderrLog were mutation-checked. --- cmd/archie-state-store/recovery.go | 24 ++-- cmd/archie-state-store/recovery_test.go | 111 +++++++++++++++++- docs/architecture/safe-change-and-recovery.md | 2 +- internal/app/archied/bootstrap.go | 13 ++ internal/app/archied/main.go | 11 +- internal/app/archied/state_store_recovery.go | 21 ++-- internal/app/controlplane/registry.go | 39 +++++- internal/app/controlplane/runtime_config.go | 31 ++++- 8 files changed, 215 insertions(+), 37 deletions(-) diff --git a/cmd/archie-state-store/recovery.go b/cmd/archie-state-store/recovery.go index 4b537e9ff..259410236 100644 --- a/cmd/archie-state-store/recovery.go +++ b/cmd/archie-state-store/recovery.go @@ -2,25 +2,20 @@ package main import ( "context" + "errors" "flag" "fmt" "io" - "os" - "path/filepath" "github.com/samcharles93/archie-core/internal/app/archied" ) -// defaultConfigPath is the configuration the serve path reads by default; the -// offline validate asks what the daemon would boot with, so it reads the same -// one. -func defaultConfigPath() string { - base, err := os.UserConfigDir() - if err != nil { - return "" - } - return filepath.Join(base, "archie", "config.toml") -} +// defaultConfigPath is the configuration the serve path reads by default, and +// the one the daemon boots with. It is the daemon's own rule rather than a +// second derivation of it: os.UserConfigDir (what the sibling processes use) +// disagrees with archied on darwin and rejects a relative XDG_CONFIG_HOME +// outright, which would leave validate answering about a file nobody boots. +func defaultConfigPath() string { return archied.DefaultConfigPath() } // recoveryUsage is the whole flag surface of the offline recovery commands. // They are subcommands of this binary rather than a recovery binary of their @@ -77,6 +72,11 @@ func runRecovery(args []string, stdout, stderr io.Writer) int { flags.StringVar(&options.Overlay, "config-overlay", "", "configuration overlay file or directory") } if err := flags.Parse(args[1:]); err != nil { + // A requested help is not a wrong command line: the usage above is the + // answer, and the serve path exits 0 for the same request. + if errors.Is(err, flag.ErrHelp) { + return 0 + } return 2 } if flags.NArg() > 0 { diff --git a/cmd/archie-state-store/recovery_test.go b/cmd/archie-state-store/recovery_test.go index 187b89762..a775037e1 100644 --- a/cmd/archie-state-store/recovery_test.go +++ b/cmd/archie-state-store/recovery_test.go @@ -317,16 +317,94 @@ func TestRecoveryValidateRunsTheGateBootRuns(t *testing.T) { t.Errorf("refusal must name what boot rejects; stderr = %q", stderr) } - // A store the State Store never seeded: the daemon cannot layer its - // settings from it either, so the offline check must not call it valid. + // A store the State Store never seeded holds no kinds at all. Its next start + // creates the resources table and seeds every kind from this same config + // (internal/app/archied/state_store.go), so archied layers those seeds and + // starts: refusing the file here would send an operator to restore a + // snapshot for a store the daemon boots on perfectly well. unseeded := filepath.Join(dir, "unseeded.db-tasks.sqlite") openTaskStore(t, unseeded) code, stdout, stderr = runRecoveryCmd(t, "validate", "-db", unseeded, "-config", configPath) - if code == 0 { - t.Fatalf("validate accepted a store with no stored settings; stdout = %q", stdout) + if code != 0 { + t.Fatalf("validate refused a store the State Store would seed on its next start; stderr = %q", stderr) } - if !strings.Contains(stderr, "provider-settings") { - t.Errorf("refusal must name the resource kind boot cannot read; stderr = %q", stderr) + if !strings.Contains(stdout, "0 stored resources validate") { + t.Errorf("validate must report that there was nothing stored to check; stdout = %q", stdout) + } + + // The upgrade case: a release defines a kind the store does not hold yet, + // and validate runs before the State Store has started again. The State + // Store seeds the kind it is missing, so this is the same stance as the + // store with nothing stored at all -- the file is not the reason the daemon + // would refuse to start, and saying otherwise costs the operator a snapshot. + partial := filepath.Join(dir, "partial.db-tasks.sqlite") + seedStoreResources(t, openTaskStore(t, partial), configPath) + if err := execRaw(t, partial, `DELETE FROM resources WHERE kind='provider-settings'`); err != nil { + t.Fatal(err) + } + code, stdout, stderr = runRecoveryCmd(t, "validate", "-db", partial, "-config", configPath) + if code != 0 { + t.Fatalf("validate refused a store missing a kind the State Store seeds; stderr = %q", stderr) + } + if count := storedResourceCount(t, stdout); count == 0 { + t.Errorf("validate must still check the kinds the store does hold; stdout = %q", stdout) + } +} + +// The subcommands advertise -h as the way to read their flags, and the serve +// path in the same binary exits 0 for the same request, so a script probing the +// recovery surface must not read a requested help as a failure. +func TestRecoveryHelpExitsZero(t *testing.T) { + commands := []string{archied.RecoveryBackup, archied.RecoveryRestore, archied.RecoveryValidate, archied.RecoveryRollback} + for _, command := range commands { + code, _, stderr := runRecoveryCmd(t, command, "-h") + if code != 0 { + t.Errorf("%s -h exited %d, want 0; stderr = %q", command, code, stderr) + } + } +} + +// validate diagnoses a deployment; it is not the deployment. Resolving the boot +// config must not create, append to, or rotate the daemon's log file or the +// directory it lives in: a line stamped component="daemon" in archied's log is +// indistinguishable from the daemon having written it. +func TestRecoveryValidateLeavesTheDaemonLogAlone(t *testing.T) { + dir := t.TempDir() + configPath, logFile := writeConfigWithLogFile(t, dir) + db := filepath.Join(dir, "archie.db-tasks.sqlite") + seedStoreResources(t, openTaskStore(t, db), configPath) + + code, _, stderr := runRecoveryCmd(t, "validate", "-db", db, "-config", configPath) + if code != 0 { + t.Fatalf("validate exited %d: %s", code, stderr) + } + if _, err := os.Stat(logFile); !os.IsNotExist(err) { + t.Fatalf("validate touched the daemon's log file %s (stat err = %v)", logFile, err) + } + if _, err := os.Stat(filepath.Dir(logFile)); !os.IsNotExist(err) { + t.Errorf("validate created the log directory %s (stat err = %v)", filepath.Dir(logFile), err) + } +} + +// validate's verdict is the daemon's verdict only if it reads the configuration +// the daemon reads. archied resolves that default from its own configHome, which +// honours a relative XDG_CONFIG_HOME; os.UserConfigDir rejects a relative one +// outright, which left the command with no config and no verdict at all. +func TestRecoveryDefaultConfigFollowsTheDaemonsConfigHome(t *testing.T) { + relative := filepath.Join("relative", "config") + t.Setenv("XDG_CONFIG_HOME", relative) + if got, want := defaultConfigPath(), filepath.Join(relative, "archie", "config.toml"); got != want { + t.Errorf("defaultConfigPath() with a relative XDG_CONFIG_HOME = %q, want %q", got, want) + } + + absolute := t.TempDir() + t.Setenv("XDG_CONFIG_HOME", absolute) + if got, want := defaultConfigPath(), filepath.Join(absolute, "archie", "config.toml"); got != want { + t.Errorf("defaultConfigPath() = %q, want %q", got, want) + } + // The equality itself: one helper, so the two cannot drift apart again. + if got, want := defaultConfigPath(), archied.DefaultConfigPath(); got != want { + t.Errorf("defaultConfigPath() = %q, but the daemon resolves %q", got, want) } } @@ -582,6 +660,27 @@ func TestRecoveryRejectsAnUnknownSubcommand(t *testing.T) { } } +// writeConfigWithLogFile is writeMinimalConfig plus the daemon's durable log +// destination, which only the daemon may bring into existence. +func writeConfigWithLogFile(t *testing.T, dir string) (string, string) { + t.Helper() + configPath := writeMinimalConfig(t, dir) + logFile := filepath.Join(dir, "logs", "archied.log") + f, err := os.OpenFile(configPath, os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + t.Fatalf("open config to append: %v", err) + } + defer func() { + if err := f.Close(); err != nil { + t.Fatalf("close config: %v", err) + } + }() + if _, err := fmt.Fprintf(f, "\n[log]\nfile = %q\n", logFile); err != nil { + t.Fatalf("append [log]: %v", err) + } + return configPath, logFile +} + // seedStoreResources seeds the store the way the State Store seeds it at boot, // from the same config file the operator hands validate: the command's verdict // is about the pair, and a store whose settings never matched the config is a diff --git a/docs/architecture/safe-change-and-recovery.md b/docs/architecture/safe-change-and-recovery.md index d9b409f0b..a1bb55e4e 100644 --- a/docs/architecture/safe-change-and-recovery.md +++ b/docs/architecture/safe-change-and-recovery.md @@ -135,7 +135,7 @@ verdict the daemon's verdict. | --- | --- | --- | | `backup -db F -out F` | writes a transactionally consistent snapshot (`VACUUM INTO`), verified as a usable store before it is published | may be running | | `restore -db F -from F` | replaces the database with a snapshot and drops the replaced file's `-wal`/`-shm` | must be stopped | -| `validate -db F [-config C]` | runs the checks that stop archied starting: the file is readable, uncorrupted and no newer than the binary; every stored resource decodes under the definition that owns it; and `configuration.Validate` accepts the stored values layered onto `C`, the check the daemon reports as `validate database settings` | may be running | +| `validate -db F [-config C]` | runs the checks that stop archied starting: the file is readable, uncorrupted and no newer than the binary; every stored resource decodes under the definition that owns it; and `configuration.Validate` accepts the settings the daemon would read -- the stored ones, or the seed the State Store writes for a kind the file does not hold yet -- layered onto `C`, the check the daemon reports as `validate database settings` | may be running | | `rollback -db F -kind K [-revision N]` | replays an earlier revision of a stored resource through the ordinary replace, recording the rollback as a new revision | must be stopped | The two validation layers `validate` reports are not the same check, and it diff --git a/internal/app/archied/bootstrap.go b/internal/app/archied/bootstrap.go index 81d22d283..bc67a1147 100644 --- a/internal/app/archied/bootstrap.go +++ b/internal/app/archied/bootstrap.go @@ -83,6 +83,13 @@ type boot struct { cfg config.Config log *slog.Logger + // stderrLog keeps setupLogging on stderr, never touching cfg.Log.File. An + // offline command reads the bootstrap config but is not the daemon: creating + // the deployment's log, appending a line stamped component="daemon", or + // rotating a log the daemon is writing are side effects a diagnosis must not + // have, and it needs no durable copy of its own. + stderrLog bool + loader *configuration.Loader doc *configuration.Document currentProvenance atomic.Pointer[configuration.Provenance] @@ -297,6 +304,12 @@ func (b *boot) loadConfig(ctx context.Context, cfgPath, overlayPath string) erro // must not take the daemon down with it. func (b *boot) setupLogging() error { cfg := b.cfg + if b.stderrLog { + // The stderr logger newBootstrap installed is the whole destination. + // Nothing below this line is set up: the feed, the task-log registry and + // the rotating file all belong to a running daemon. + return nil + } logFeed := logging.NewFeed(1000) b.logFeed = logFeed // Task logs live alongside the store rather than under cfg.Log.File's diff --git a/internal/app/archied/main.go b/internal/app/archied/main.go index 37870ec3f..3ea12fbe2 100644 --- a/internal/app/archied/main.go +++ b/internal/app/archied/main.go @@ -315,7 +315,7 @@ type runArgs struct { } func parseArgs() (runArgs, bool) { - defaultCfg := filepath.Join(configHome(), "archie", "config.toml") + defaultCfg := DefaultConfigPath() var args runArgs flag.StringVar(&args.cfgPath, "config", defaultCfg, "path to a TOML/YAML config file or configuration directory") flag.StringVar(&args.overlayPath, "config-overlay", "", "path to a TOML/YAML overlay file or configuration directory applied on top of -config") @@ -918,6 +918,15 @@ func configHome() string { return filepath.Join(home, ".config") } +// DefaultConfigPath is the configuration the daemon reads when -config is not +// given, and the one every command that asks a question about the daemon has to +// read to be answering about the same deployment: it is the daemon's rule, not +// a second one. It is exported for exactly that reason -- an offline command +// resolving the default any other way answers about a different config file +// (os.UserConfigDir, notably, rejects a relative XDG_CONFIG_HOME outright and +// leaves the caller with no path at all). +func DefaultConfigPath() string { return filepath.Join(configHome(), "archie", "config.toml") } + // updateReportPath is where the update watchdog leaves the phase-2 outcome // of an update for this identity to relay on its next launch. The identity is // hashed so multiple identities sharing one daemon (see diff --git a/internal/app/archied/state_store_recovery.go b/internal/app/archied/state_store_recovery.go index 868755366..92b371d1f 100644 --- a/internal/app/archied/state_store_recovery.go +++ b/internal/app/archied/state_store_recovery.go @@ -132,10 +132,12 @@ func validateStore(ctx context.Context, options StateStoreRecoveryOptions) (stri defer func() { _ = st.Close() }() server := controlplane.NewServer(st) - // The resources table is only absent from a store written before the control - // plane existed. That store holds no settings to validate, and the serving - // process creates the table on its next start -- refusing it here would send - // an operator to restore a snapshot for nothing. + // Is there a resources table at all? A store written before the control plane + // existed has none, and the State Store creates it and seeds every kind from + // this same config on its next start -- so there is nothing stored to check + // with the writer's validator, and boot's gate is the config check alone. + // That is the same stance the layering takes for a kind the table is missing: + // what the daemon reads for a kind nothing was ever stored for is the seed. checked := 0 stored, err := st.StoresResources(ctx) if err != nil { @@ -153,10 +155,8 @@ func validateStore(ctx context.Context, options StateStoreRecoveryOptions) (stri } document := base if stored { - // A store the State Store never seeded has no kinds to layer, and the - // daemon cannot start against it either: the State Store seeds every kind - // from its own config at boot. - if document, _, err = server.StoredRuntimeConfig(ctx, base); err != nil { + document, _, err = server.StoredRuntimeConfig(ctx, base) + if err != nil { return "", err } } @@ -172,8 +172,13 @@ func validateStore(ctx context.Context, options StateStoreRecoveryOptions) (stri // the only recovery command that asks a question about the process rather than // the file, and that question is about the stored values layered onto this // document, so the document is part of the answer. +// +// It reads that config with the daemon's stderr logger: resolving it is a +// diagnosis, and a diagnosis that opened, appended to or rotated cfg.Log.File +// would edit the deployment it is inspecting. func bootConfig(ctx context.Context, options StateStoreRecoveryOptions) (config.Config, error) { b := newBootstrap() + b.stderrLog = true defer b.cleanup() if err := b.loadConfig(ctx, options.Config, options.Overlay); err != nil { return config.Config{}, err diff --git a/internal/app/controlplane/registry.go b/internal/app/controlplane/registry.go index bbfa206e5..629421c93 100644 --- a/internal/app/controlplane/registry.go +++ b/internal/app/controlplane/registry.go @@ -72,13 +72,9 @@ func (s *Server) ImportConfig(ctx context.Context, cfg config.Config) (map[strin if !errors.Is(err, store.ErrResourceNotFound) { return nil, err } - value, err := json.Marshal(definition.Seed(cfg)) + value, err := definition.seededValue(cfg) if err != nil { - return nil, fmt.Errorf("seed %s: %w", definition.Kind, err) - } - value, err = definition.Decode(value) - if err != nil { - return nil, fmt.Errorf("seed %s: %w", definition.Kind, err) + return nil, err } resource, err = s.store.PutResource(ctx, store.ResourceWrite{Kind: definition.Kind, Value: value, Actor: "system:migration", Source: "legacy-config", RequestID: "import:" + definition.Kind, ExpectedVersion: 0, At: time.Now().UTC()}) if err != nil { @@ -89,6 +85,37 @@ func (s *Server) ImportConfig(ctx context.Context, cfg config.Config) (map[strin return versions, nil } +// seededValue is the value the State Store writes for a kind it does not hold +// yet: the definition's seed derived from cfg, run through the same Decode a +// write applies. ImportConfig stores it; the offline config layering reads it +// for the same reason, so "what the daemon sees for a kind that was never +// stored" is one value rather than two. +func (d Definition) seededValue(cfg config.Config) ([]byte, error) { + value, err := json.Marshal(d.Seed(cfg)) + if err != nil { + return nil, fmt.Errorf("seed %s: %w", d.Kind, err) + } + value, err = d.Decode(value) + if err != nil { + return nil, fmt.Errorf("seed %s: %w", d.Kind, err) + } + return value, nil +} + +// seededValues is what the State Store would write for every kind, keyed by +// kind. +func (s *Server) seededValues(cfg config.Config) (map[string][]byte, error) { + values := make(map[string][]byte, len(s.ordered)) + for _, definition := range s.ordered { + value, err := definition.seededValue(cfg) + if err != nil { + return nil, err + } + values[definition.Kind] = value + } + return values, nil +} + // Owns reports whether kind is a resource the control plane owns, so a caller // acting on a stored value can refuse a kind nobody defines before it reaches // the store. diff --git a/internal/app/controlplane/runtime_config.go b/internal/app/controlplane/runtime_config.go index 93fb6f9d2..0bd90c6c5 100644 --- a/internal/app/controlplane/runtime_config.go +++ b/internal/app/controlplane/runtime_config.go @@ -3,11 +3,13 @@ package controlplane import ( "context" "encoding/json" + "errors" "fmt" "time" "github.com/samcharles93/archie-core/internal/config" pb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1" + "github.com/samcharles93/archie-core/internal/store" ) // resourceReader is the one read the layering performs: a resource kind's value @@ -43,19 +45,42 @@ func (c *Client) RuntimeChatConfig(ctx context.Context, base config.ChatConfig) // configuration.Validate, so an offline check of that database has to make the // same first move with the same implementation; called with the config the // daemon would load, the result is the document boot decides on. +// +// A kind the store does not hold is read as the seed the State Store would +// write for it from base (see storeReader.query), because that store seeds +// every kind before it serves. func (s *Server) StoredRuntimeConfig(ctx context.Context, base config.Config) (config.Config, map[string]int64, error) { - return runtimeConfigFrom(ctx, storeReader{resources: s.store}, base) + seeds, err := s.seededValues(base) + if err != nil { + return config.Config{}, nil, err + } + return runtimeConfigFrom(ctx, storeReader{resources: s.store, seeds: seeds}, base) } // storeReader reads the server's own store. It mirrors what the gRPC read -// answers -- a missing kind is an error, not an empty value -- so a store the -// daemon cannot layer settings from is one this reports rather than skips. +// answers -- a kind that is stored is the value and version it holds -- with +// one difference: a kind the store does not hold yet is the value the State +// Store seeds for it, since the daemon dials a store that has already run +// ImportConfig over the same config. type storeReader struct { resources ResourceStore + seeds map[string][]byte } func (r storeReader) query(ctx context.Context, kind string, decode func([]byte) error) (int64, error) { resource, err := r.resources.Resource(ctx, kind) + if errors.Is(err, store.ErrResourceNotFound) { + seed, ok := r.seeds[kind] + if !ok { + return 0, fmt.Errorf("read %s: %w", kind, store.ErrResourceNotFound) + } + if err := decode(seed); err != nil { + return 0, fmt.Errorf("decode %s seed: %w", kind, err) + } + // No version: nothing has been stored for this kind, so nothing has + // been applied either. + return 0, nil + } if err != nil { return 0, fmt.Errorf("read %s: %w", kind, err) } From 21c6aca5136ba1a9d287c174b26282a803e83d74 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Mon, 21 Sep 2026 22:07:41 +1000 Subject: [PATCH 4/4] fix(statestore): build the recovery control-plane server on the shared step vocabulary After the rebase onto main, controlplane.NewServer takes the step vocabulary it resolves definitions against (controlplane.ResourceStore, *workflow.Manager) and returns (Server, error). Both offline call sites still passed the store alone, so internal/app/archied stopped compiling -- and with it cmd/archie-state-store, cmd/archied and cmd/archie-gateway. validate and rollback now build their server through openStateStoreControlPlane, the root helper the served path already uses, so the offline checks resolve the same shared provider set (internal/infrastructure/workflowsteps) as the process whose verdict they predict, and the constructor's error is returned as a refusal rather than dropped. No second vocabulary call site: the fwmp pin on openStateStoreControlPlane covers these paths unchanged. The test's seeding helper builds its server from the same shared set (workflowsteps.NewManager) because it cannot call the unexported root helper; an empty manager there would seed values the served path never validates against. That is not merely a claim: with an empty manager the three validate tests fail with `workflow "bootstrap" step 1: unknown type "bootstrap.prepare"`, so they already pin that the offline path resolves a live, shared vocabulary. --- cmd/archie-state-store/recovery_test.go | 19 ++++++++++++++++++- internal/app/archied/state_store_recovery.go | 17 ++++++++++++++--- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/cmd/archie-state-store/recovery_test.go b/cmd/archie-state-store/recovery_test.go index a775037e1..b58629f29 100644 --- a/cmd/archie-state-store/recovery_test.go +++ b/cmd/archie-state-store/recovery_test.go @@ -27,6 +27,7 @@ import ( "github.com/samcharles93/archie-core/internal/app/archied" "github.com/samcharles93/archie-core/internal/app/controlplane" "github.com/samcharles93/archie-core/internal/infrastructure/configuration" + "github.com/samcharles93/archie-core/internal/infrastructure/workflowsteps" "github.com/samcharles93/archie-core/internal/store" ) @@ -685,13 +686,29 @@ func writeConfigWithLogFile(t *testing.T, dir string) (string, string) { // from the same config file the operator hands validate: the command's verdict // is about the pair, and a store whose settings never matched the config is a // state boot does not recognise either. +// +// The seeding builds its control-plane server from the shared provider set +// (internal/infrastructure/workflowsteps), which is what every composition root +// registers -- including openStateStoreControlPlane, the root helper the +// recovery path uses. This test cannot call that unexported helper, so it +// registers the same set itself rather than an empty manager: the definitions +// are decoded with the vocabulary the served path validates against, which is +// what makes the seeded values the ones validate is asked about. func seedStoreResources(t *testing.T, st *store.Store, configPath string) { t.Helper() doc, err := configuration.New(nil).Resolve(configPath, "") if err != nil { t.Fatalf("resolve %s: %v", configPath, err) } - if _, err := controlplane.NewServer(st).ImportConfig(t.Context(), doc.Config); err != nil { + steps, err := workflowsteps.NewManager() + if err != nil { + t.Fatalf("register workflow step vocabulary: %v", err) + } + server, err := controlplane.NewServer(st, steps) + if err != nil { + t.Fatalf("build control plane server: %v", err) + } + if _, err := server.ImportConfig(t.Context(), doc.Config); err != nil { t.Fatalf("seed store resources: %v", err) } } diff --git a/internal/app/archied/state_store_recovery.go b/internal/app/archied/state_store_recovery.go index 92b371d1f..fe4636726 100644 --- a/internal/app/archied/state_store_recovery.go +++ b/internal/app/archied/state_store_recovery.go @@ -11,6 +11,12 @@ // cannot stop the process that runs it, so the snapshot that makes a failed // update reversible is taken against a serving store through SQLite's own // consistent copy. +// +// The control-plane server both validate and rollback read through is built by +// openStateStoreControlPlane -- the same root helper the served path uses -- so +// the offline checks resolve the same workflow step vocabulary as the process +// whose verdict they predict, and a constructor error is a refusal here rather +// than a server that would validate against a different vocabulary. package archied import ( @@ -19,7 +25,6 @@ import ( "fmt" "time" - "github.com/samcharles93/archie-core/internal/app/controlplane" "github.com/samcharles93/archie-core/internal/config" pb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1" "github.com/samcharles93/archie-core/internal/infrastructure/configuration" @@ -131,7 +136,10 @@ func validateStore(ctx context.Context, options StateStoreRecoveryOptions) (stri } defer func() { _ = st.Close() }() - server := controlplane.NewServer(st) + server, err := openStateStoreControlPlane(st) + if err != nil { + return "", err + } // Is there a resources table at all? A store written before the control plane // existed has none, and the State Store creates it and seeds every kind from // this same config on its next start -- so there is nothing stored to check @@ -210,7 +218,10 @@ func rollbackResource(ctx context.Context, options StateStoreRecoveryOptions) (s } defer func() { _ = st.Close() }() - server := controlplane.NewServer(st) + server, err := openStateStoreControlPlane(st) + if err != nil { + return "", err + } if !server.Owns(options.Kind) { return "", fmt.Errorf("unknown resource kind %q", options.Kind) }