diff --git a/cmd/archie-state-store/main.go b/cmd/archie-state-store/main.go index 130956aad..6ca5ba8a5 100644 --- a/cmd/archie-state-store/main.go +++ b/cmd/archie-state-store/main.go @@ -15,6 +15,7 @@ import ( "os" "os/signal" "path/filepath" + "strings" "syscall" "github.com/samcharles93/archie-core/internal/app/archied" @@ -23,6 +24,13 @@ import ( func main() { os.Exit(run()) } func run() int { + // A leading non-flag argument selects a recovery subcommand; anything else, + // including no arguments at all, serves the State Store contract. An + // unrecognised word is refused rather than ignored, so a mistyped recovery + // command never leaves an operator watching a server start instead. + if args := os.Args[1:]; len(args) > 0 && !strings.HasPrefix(args[0], "-") { + return runRecovery(args, os.Stdout, os.Stderr) + } base, err := os.UserConfigDir() if err != nil { fmt.Fprintln(os.Stderr, err) @@ -34,6 +42,10 @@ func run() int { flag.StringVar(&options.Listen, "listen", "", "state store gRPC listen address (defaults to [services.state].listen, else 127.0.0.1:9090)") flag.StringVar(&options.Token, "token", "", "bearer token required for a non-loopback listener (defaults to [services.state].target_token)") flag.StringVar(&options.ReadyAddr, "ready-addr", "", "optional readiness HTTP listen address (e.g. 127.0.0.1:9091)") + flag.Usage = func() { + fmt.Fprintf(flag.CommandLine.Output(), "usage: archie-state-store [flags] # serve the State Store gRPC contract\n\noffline recovery: archie-state-store -h\n\n") + flag.PrintDefaults() + } flag.Parse() ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() diff --git a/cmd/archie-state-store/recovery.go b/cmd/archie-state-store/recovery.go new file mode 100644 index 000000000..00ee75750 --- /dev/null +++ b/cmd/archie-state-store/recovery.go @@ -0,0 +1,74 @@ +package main + +import ( + "context" + "flag" + "fmt" + "io" + + "github.com/samcharles93/archie-core/internal/app/archied" +) + +// recoveryUsage is the whole flag surface of the offline recovery commands. +// They are subcommands of this binary rather than a recovery binary of their +// own: offline database maintenance belongs to the process that owns the file +// and its path layout (docs/architecture/organisation.md#process-boundaries). +const recoveryUsage = `usage: archie-state-store [flags] + +With no command, archie-state-store serves the State Store gRPC contract. + +The recovery commands operate on the task database file directly -- the +configured db_path with "-tasks.sqlite" appended -- and are how an operator +recovers a database the control plane will not start on. Stop the State Store +first: restore and rollback rewrite the file and refuse while another process +owns it, while backup takes a consistent snapshot of a serving store. + + backup -db FILE -out FILE write a snapshot of the database + restore -db FILE -from FILE replace the database with a snapshot + validate -db FILE check the database the way boot does + rollback -db FILE -kind KIND replay an earlier revision of a stored + [-revision N] resource through the ordinary replace +` + +// runRecovery parses and performs one offline recovery command, reporting the +// operation's summary on stdout. Exit codes follow the serve path: 0 succeeded, +// 1 the operation failed, 2 the command line was wrong. +func runRecovery(args []string, stdout, stderr io.Writer) int { + if len(args) == 0 { + fmt.Fprint(stderr, recoveryUsage) + return 2 + } + command := args[0] + options := archied.StateStoreRecoveryOptions{Operation: command} + switch command { + case archied.RecoveryBackup, archied.RecoveryRestore, archied.RecoveryValidate, archied.RecoveryRollback: + default: + fmt.Fprintf(stderr, "archie-state-store: unknown command %q\n\n", command) + fmt.Fprint(stderr, recoveryUsage) + return 2 + } + + flags := flag.NewFlagSet("archie-state-store "+command, flag.ContinueOnError) + flags.SetOutput(stderr) + flags.Usage = func() { fmt.Fprint(stderr, recoveryUsage) } + flags.StringVar(&options.DB, "db", "", "task database file the State Store owns (-tasks.sqlite)") + flags.StringVar(&options.Out, "out", "", "snapshot file backup writes") + flags.StringVar(&options.From, "from", "", "snapshot file restore reads") + flags.StringVar(&options.Kind, "kind", "", "control-plane resource kind rollback replays") + flags.Int64Var(&options.Revision, "revision", 0, "revision rollback replays (default: the newest one older than the current value)") + if err := flags.Parse(args[1:]); err != nil { + return 2 + } + if flags.NArg() > 0 { + fmt.Fprintf(stderr, "archie-state-store %s: unexpected argument %q\n", command, flags.Arg(0)) + return 2 + } + + summary, err := archied.RunStateStoreRecovery(context.Background(), options) + if err != nil { + fmt.Fprintf(stderr, "archie-state-store %s: %v\n", command, err) + return 1 + } + fmt.Fprintln(stdout, summary) + return 0 +} diff --git a/cmd/archie-state-store/recovery_test.go b/cmd/archie-state-store/recovery_test.go new file mode 100644 index 000000000..efe96c4f9 --- /dev/null +++ b/cmd/archie-state-store/recovery_test.go @@ -0,0 +1,544 @@ +// Package main tests the offline recovery subcommands of archie-state-store: +// backup, restore, validate and rollback against the task database file +// directly. They are the operator's only path back when the control plane's +// stored settings will not validate, because the daemon fails closed: the +// in-band remedy (replay an earlier revision while the State Store is up) is +// unavailable in exactly the case that needs it. +// +// The tests drive the subcommands the way an operator does -- through +// runRecovery, against real files on disk -- rather than through the internal +// helpers, so the flag surface and the exit codes are part of what is verified. + +package main + +import ( + "bytes" + "database/sql" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + _ "modernc.org/sqlite" + + "github.com/samcharles93/archie-core/internal/app/archied" + "github.com/samcharles93/archie-core/internal/app/controlplane" + "github.com/samcharles93/archie-core/internal/store" +) + +// runRecoveryCmd runs one subcommand the way main does and returns its exit +// code with both streams captured. +func runRecoveryCmd(t *testing.T, args ...string) (int, string, string) { + t.Helper() + var stdout, stderr bytes.Buffer + code := runRecovery(args, &stdout, &stderr) + return code, stdout.String(), stderr.String() +} + +// openTaskStore opens the task database the subcommands operate on. It does not +// take the ownership lock: only the serving process does that, and a test that +// wants a "live" store holds it explicitly with holdStoreOwnership. +func openTaskStore(t *testing.T, path string) *store.Store { + t.Helper() + st, err := store.Open(t.Context(), path) + if err != nil { + t.Fatalf("open task store %s: %v", path, err) + } + t.Cleanup(func() { _ = st.Close() }) + return st +} + +// holdStoreOwnership takes the ownership lock the running State Store holds for +// its whole life, so a test can prove a command refuses to touch a live store. +func holdStoreOwnership(t *testing.T, path string) { + t.Helper() + ownership, err := store.AcquireOwnership(path) + if err != nil { + t.Fatalf("acquire store ownership: %v", err) + } + t.Cleanup(func() { _ = ownership.Release() }) +} + +// seedTask writes one task so a snapshot can be distinguished from the live +// database by what it contains, not merely by its existence. +func seedTask(t *testing.T, st *store.Store, title string) { + t.Helper() + if _, err := st.EnqueueChatTask(t.Context(), "acme", "widget", title, "body", "implement", ""); err != nil { + t.Fatalf("seed task %q: %v", title, err) + } +} + +func taskTitles(t *testing.T, path string) []string { + t.Helper() + st := openTaskStore(t, path) + tasks, err := st.Tasks(t.Context(), 100) + if err != nil { + t.Fatalf("list tasks in %s: %v", path, err) + } + titles := make([]string, 0, len(tasks)) + for _, task := range tasks { + titles = append(titles, task.Title) + } + return titles +} + +// backupPath is the snapshot every restore test restores from. +func backupPath(t *testing.T, dir string) string { return filepath.Join(dir, "snapshot.sqlite") } + +// backup takes the snapshot the caller then restores, through the subcommand +// under test rather than through the store directly, so a broken backup fails +// the restore test too instead of silently passing an empty file. +func backup(t *testing.T, db, out string) { + t.Helper() + code, _, stderr := runRecoveryCmd(t, "backup", "-db", db, "-out", out) + if code != 0 { + t.Fatalf("backup exited %d: %s", code, stderr) + } +} + +// The update installer cannot stop the process that runs it, so it backs the +// task database up while the State Store is serving. VACUUM INTO is safe +// against a live writer; a backup that insisted on an exclusively-owned file +// would break the one caller it has, and the update would refuse to start. +func TestRecoveryBackupSnapshotsAStoreItsStateStoreIsServing(t *testing.T) { + dir := t.TempDir() + db := filepath.Join(dir, "archie.db-tasks.sqlite") + out := backupPath(t, dir) + seedTask(t, openTaskStore(t, db), "before") + holdStoreOwnership(t, db) + if err := os.WriteFile(out, []byte("stale snapshot"), 0o600); err != nil { + t.Fatal(err) + } + + code, stdout, stderr := runRecoveryCmd(t, "backup", "-db", db, "-out", out) + if code != 0 { + t.Fatalf("backup exited %d: %s", code, stderr) + } + if !strings.Contains(stdout, out) || !strings.Contains(stdout, db) { + t.Errorf("backup must report the database and the snapshot it wrote; stdout = %q", stdout) + } + // The stale file was replaced by a real snapshot, and the snapshot is a + // store a restore can put back: same rows, readable through the store. + if got := taskTitles(t, out); len(got) != 1 || got[0] != "before" { + t.Fatalf("snapshot tasks = %v, want the one seeded task", got) + } + if entries, err := filepath.Glob(out + ".tmp*"); err != nil || len(entries) != 0 { + t.Errorf("backup left scratch files beside the snapshot: %v (%v)", entries, err) + } +} + +// The snapshot is the rollback state, so an interrupted backup must never +// leave a truncated file where a good snapshot was. +func TestRecoveryBackupRefusesAMissingDatabase(t *testing.T) { + dir := t.TempDir() + code, _, stderr := runRecoveryCmd(t, "backup", "-db", filepath.Join(dir, "absent.sqlite"), "-out", backupPath(t, dir)) + if code == 0 { + t.Fatal("backup of a missing database succeeded") + } + if !strings.Contains(stderr, "absent.sqlite") { + t.Errorf("refusal must name the database it could not read; stderr = %q", stderr) + } +} + +// Restoring is the forward-migration escape hatch: the previous release +// reads this file, so it must be exactly the snapshot, with the WAL that +// belongs to the replaced database gone rather than replayed over it. +func TestRecoveryRestorePutsTheSnapshotBackAndDropsItsWAL(t *testing.T) { + dir := t.TempDir() + db := filepath.Join(dir, "archie.db-tasks.sqlite") + out := backupPath(t, dir) + st := openTaskStore(t, db) + seedTask(t, st, "before") + backup(t, db, out) + seedTask(t, st, "after") + + code, stdout, stderr := runRecoveryCmd(t, "restore", "-db", db, "-from", out) + if code != 0 { + t.Fatalf("restore exited %d: %s", code, stderr) + } + if !strings.Contains(stdout, out) { + t.Errorf("restore must report the snapshot it restored from; stdout = %q", stdout) + } + // Checked before anything reopens the database: opening a store sets WAL + // mode, which creates these files, so the point is that the restore did not + // leave the replaced database's log behind -- not that a later reader + // cannot recreate its own. + for _, suffix := range []string{"-wal", "-shm"} { + if _, err := os.Stat(db + suffix); err == nil { + t.Errorf("%s survived the restore: the replaced database's write-ahead log must not be replayed over the snapshot", db+suffix) + } + } + if got := taskTitles(t, db); len(got) != 1 || got[0] != "before" { + t.Fatalf("restored tasks = %v, want only the snapshot's task", got) + } +} + +// Overwriting a database a running State Store still owns leaves that process +// writing to an unlinked inode while the store serves a file nobody owns -- +// the one way an offline command can destroy a task store. +func TestRecoveryRestoreRefusesAStoreTheStateStoreIsRunning(t *testing.T) { + dir := t.TempDir() + db := filepath.Join(dir, "archie.db-tasks.sqlite") + out := backupPath(t, dir) + st := openTaskStore(t, db) + seedTask(t, st, "before") + backup(t, db, out) + seedTask(t, st, "after") + holdStoreOwnership(t, db) + + code, _, stderr := runRecoveryCmd(t, "restore", "-db", db, "-from", out) + if code == 0 { + t.Fatal("restore against a live State Store succeeded") + } + if !strings.Contains(stderr, "owned by another process") { + t.Errorf("refusal must say the database is owned by another process; stderr = %q", stderr) + } + if got := taskTitles(t, db); len(got) != 2 { + t.Fatalf("refused restore changed the database: tasks = %v", got) + } +} + +func TestRecoveryRestoreRefusesAFileThatIsNotAStore(t *testing.T) { + dir := t.TempDir() + db := filepath.Join(dir, "archie.db-tasks.sqlite") + seedTask(t, openTaskStore(t, db), "live") + notAStore := filepath.Join(dir, "notes.txt") + if err := os.WriteFile(notAStore, []byte("this is not a database"), 0o600); err != nil { + t.Fatal(err) + } + + code, _, stderr := runRecoveryCmd(t, "restore", "-db", db, "-from", notAStore) + if code == 0 { + t.Fatal("restore from a file that is not a store succeeded") + } + if !strings.Contains(stderr, notAStore) { + t.Errorf("refusal must name the snapshot it could not read; stderr = %q", stderr) + } + if got := taskTitles(t, db); len(got) != 1 { + t.Fatalf("refused restore changed the database: tasks = %v", got) + } +} + +func TestRecoveryRestoreRefusesTheDatabaseAsItsOwnSnapshot(t *testing.T) { + // Restoring a database from itself would delete the write-ahead log of the + // file it is about to copy, so the one case that must never reach the swap + // is refused up front. + db := filepath.Join(t.TempDir(), "archie.db-tasks.sqlite") + seedTask(t, openTaskStore(t, db), "live") + + code, _, stderr := runRecoveryCmd(t, "restore", "-db", db, "-from", db) + if code == 0 { + t.Fatal("restore from the database itself succeeded") + } + if !strings.Contains(stderr, "is the database itself") { + t.Errorf("refusal must say the snapshot is the database; stderr = %q", stderr) + } + if got := taskTitles(t, db); len(got) != 1 { + t.Fatalf("refused restore changed the database: tasks = %v", got) + } +} + +func TestRecoveryValidateAcceptsAHealthyStore(t *testing.T) { + dir := t.TempDir() + db := filepath.Join(dir, "archie.db-tasks.sqlite") + st := openTaskStore(t, db) + seedTask(t, st, "healthy") + seedResource(t, st, controlplane.ModelRoleAssignmentsKind, `{"implement":"anthropic/claude"}`, "seed") + + code, stdout, stderr := runRecoveryCmd(t, "validate", "-db", db) + if code != 0 { + t.Fatalf("validate exited %d: %s", code, stderr) + } + if !strings.Contains(stdout, "schema version") { + t.Errorf("validate must report the schema version it verified; stdout = %q", stdout) + } +} + +// validate exists to answer "would archied start against this file". Each case +// below is a state the daemon fails closed on, or a file it cannot read at all. +func TestRecoveryValidateRefusesWhatTheDaemonRefuses(t *testing.T) { + newStore := func(t *testing.T) string { + t.Helper() + db := filepath.Join(t.TempDir(), "archie.db-tasks.sqlite") + openTaskStore(t, db) + return db + } + + t.Run("forward_migrated_schema", func(t *testing.T) { + // A database written by a newer release is the state a rollback has to + // recognise before it hands the file to an older binary. + db := newStore(t) + if err := execRaw(t, db, `PRAGMA user_version = 99`); err != nil { + t.Fatal(err) + } + code, _, stderr := runRecoveryCmd(t, "validate", "-db", db) + if code == 0 { + t.Fatal("validate accepted a forward-migrated database") + } + if !strings.Contains(stderr, "newer than supported version") { + t.Errorf("refusal must name the schema version; stderr = %q", stderr) + } + }) + + t.Run("corrupt_file", func(t *testing.T) { + // A real database header over pages that are not a database. The magic + // is right, so this fails on the content rather than on the format, which + // is the shape a half-written or truncated store has. + db := filepath.Join(t.TempDir(), "archie.db-tasks.sqlite") + corrupt := append([]byte("SQLite format 3\x00"), bytes.Repeat([]byte{0xff}, 4096)...) + if err := os.WriteFile(db, corrupt, 0o600); err != nil { + t.Fatal(err) + } + code, _, stderr := runRecoveryCmd(t, "validate", "-db", db) + if code == 0 { + t.Fatal("validate accepted a corrupt database") + } + if !strings.Contains(stderr, db) { + t.Errorf("refusal must name the file; stderr = %q", stderr) + } + }) + + t.Run("not_a_database", func(t *testing.T) { + // Pointing -db at the wrong file is the common operator mistake, and the + // configured db_path is a different file from the task database the + // State Store owns. + db := filepath.Join(t.TempDir(), "archie.db") + if err := os.WriteFile(db, []byte("db_path, not the task database"), 0o600); err != nil { + t.Fatal(err) + } + code, _, stderr := runRecoveryCmd(t, "validate", "-db", db) + if code == 0 { + t.Fatal("validate accepted a file that is not a database") + } + if !strings.Contains(stderr, db) { + t.Errorf("refusal must name the file; stderr = %q", stderr) + } + }) + + t.Run("stored_resource_that_will_not_validate", func(t *testing.T) { + // The stored settings value that stops archied starting: it can only + // get here by bypassing the write path, which is exactly the state the + // control plane fails closed on and the operator has no path back from. + db := newStore(t) + seedResource(t, openTaskStore(t, db), controlplane.ModelRoleAssignmentsKind, `{"implement":"claude"}`, "hand-edited") + code, _, stderr := runRecoveryCmd(t, "validate", "-db", db) + if code == 0 { + t.Fatal("validate accepted a stored resource the write path would refuse") + } + if !strings.Contains(stderr, controlplane.ModelRoleAssignmentsKind) { + t.Errorf("refusal must name the resource kind; stderr = %q", stderr) + } + }) + + t.Run("missing_database", func(t *testing.T) { + db := newStore(t) + if err := os.Remove(db); err != nil { + t.Fatal(err) + } + code, _, stderr := runRecoveryCmd(t, "validate", "-db", db) + if code == 0 { + t.Fatal("validate accepted a database that does not exist") + } + if !strings.Contains(stderr, db) { + t.Errorf("refusal must name the file; stderr = %q", stderr) + } + }) + + t.Run("store_without_the_control_plane_table", func(t *testing.T) { + // A store written before the control plane existed holds no stored + // settings. The serving process creates the table on its next start, so + // refusing this file would send an operator to restore a snapshot for a + // database that would have started perfectly well. + db := newStore(t) + if err := execRaw(t, db, `DROP TABLE resources`); err != nil { + t.Fatal(err) + } + code, stdout, stderr := runRecoveryCmd(t, "validate", "-db", db) + if code != 0 { + t.Fatalf("validate exited %d: %s", code, stderr) + } + if !strings.Contains(stdout, "0 stored resources validate") { + t.Errorf("validate must report that there was nothing stored to check; stdout = %q", stdout) + } + }) +} + +// rollback is the one operation that closes the documented gap: with the State +// Store stopped, it replays the value an earlier revision recorded through the +// ordinary replace, so no rollback RPC is needed and the rollback itself is +// audited as one more revision. +func TestRecoveryRollbackReplaysTheRevisionThroughReplace(t *testing.T) { + dir := t.TempDir() + db := filepath.Join(dir, "archie.db-tasks.sqlite") + st := openTaskStore(t, db) + seedResource(t, st, controlplane.ModelRoleAssignmentsKind, `{"implement":"anthropic/claude"}`, "first") + seedResource(t, st, controlplane.ModelRoleAssignmentsKind, `{"implement":"openai/gpt"}`, "second") + + code, stdout, stderr := runRecoveryCmd(t, "rollback", "-db", db, "-kind", controlplane.ModelRoleAssignmentsKind) + if code != 0 { + t.Fatalf("rollback exited %d: %s", code, stderr) + } + if !strings.Contains(stdout, controlplane.ModelRoleAssignmentsKind) { + t.Errorf("rollback must name the resource it replayed; stdout = %q", stdout) + } + resource, err := st.Resource(t.Context(), controlplane.ModelRoleAssignmentsKind) + if err != nil { + t.Fatal(err) + } + if string(resource.Value) != `{"implement":"anthropic/claude"}` { + t.Errorf("resource value = %s, want the first revision's value", resource.Value) + } + if resource.Version != 3 { + t.Errorf("resource version = %d, want a new revision (3) rather than a rewrite of the old one", resource.Version) + } + history, err := st.ResourceHistory(t.Context(), controlplane.ModelRoleAssignmentsKind, 0) + if err != nil { + t.Fatal(err) + } + if len(history) != 3 { + t.Fatalf("history has %d revisions, want the rollback recorded as the third", len(history)) + } + if history[0].Source != archied.OfflineRollbackSource { + t.Errorf("newest revision source = %q, want %q so the rollback is auditable", history[0].Source, archied.OfflineRollbackSource) + } +} + +// The revision to restore may be named, which is how an operator reaches past +// the most recent change without replaying it first. +func TestRecoveryRollbackRestoresANamedRevision(t *testing.T) { + dir := t.TempDir() + db := filepath.Join(dir, "archie.db-tasks.sqlite") + st := openTaskStore(t, db) + seedResource(t, st, controlplane.ModelRoleAssignmentsKind, `{"implement":"anthropic/claude"}`, "first") + seedResource(t, st, controlplane.ModelRoleAssignmentsKind, `{"implement":"openai/gpt"}`, "second") + seedResource(t, st, controlplane.ModelRoleAssignmentsKind, `{"implement":"google/gemini"}`, "third") + + code, _, stderr := runRecoveryCmd(t, "rollback", "-db", db, "-kind", controlplane.ModelRoleAssignmentsKind, "-revision", "1") + if code != 0 { + t.Fatalf("rollback exited %d: %s", code, stderr) + } + resource, err := st.Resource(t.Context(), controlplane.ModelRoleAssignmentsKind) + if err != nil { + t.Fatal(err) + } + if string(resource.Value) != `{"implement":"anthropic/claude"}` { + t.Errorf("resource value = %s, want the value recorded at revision 1", resource.Value) + } +} + +func TestRecoveryRollbackRefusesWhatItCannotReplay(t *testing.T) { + newStore := func(t *testing.T, revisions int) string { + t.Helper() + db := filepath.Join(t.TempDir(), "archie.db-tasks.sqlite") + st := openTaskStore(t, db) + for i := range revisions { + value := `{"implement":"anthropic/claude"}` + if i > 0 { + value = `{"implement":"openai/gpt"}` + } + seedResource(t, st, controlplane.ModelRoleAssignmentsKind, value, "seed") + } + return db + } + + t.Run("unknown_kind", func(t *testing.T) { + db := newStore(t, 2) + code, _, stderr := runRecoveryCmd(t, "rollback", "-db", db, "-kind", "not-a-resource") + if code == 0 { + t.Fatal("rollback of an unknown resource kind succeeded") + } + if !strings.Contains(stderr, "not-a-resource") { + t.Errorf("refusal must name the kind; stderr = %q", stderr) + } + }) + + t.Run("no_earlier_revision", func(t *testing.T) { + db := newStore(t, 1) + code, _, stderr := runRecoveryCmd(t, "rollback", "-db", db, "-kind", controlplane.ModelRoleAssignmentsKind) + if code == 0 { + t.Fatal("rollback with no earlier revision succeeded") + } + if !strings.Contains(stderr, "no earlier revision") { + t.Errorf("refusal must say there is nothing to roll back to; stderr = %q", stderr) + } + }) + + t.Run("current_revision", func(t *testing.T) { + db := newStore(t, 2) + code, _, stderr := runRecoveryCmd(t, "rollback", "-db", db, "-kind", controlplane.ModelRoleAssignmentsKind, "-revision", "2") + if code == 0 { + t.Fatal("rollback of the current revision succeeded") + } + if !strings.Contains(stderr, "is the current version") { + t.Errorf("refusal must say the revision is already the current value; stderr = %q", stderr) + } + }) + + t.Run("live_state_store", func(t *testing.T) { + db := newStore(t, 2) + holdStoreOwnership(t, db) + code, _, stderr := runRecoveryCmd(t, "rollback", "-db", db, "-kind", controlplane.ModelRoleAssignmentsKind) + if code == 0 { + t.Fatal("rollback against a live State Store succeeded") + } + if !strings.Contains(stderr, "owned by another process") { + t.Errorf("refusal must say the database is owned by another process; stderr = %q", stderr) + } + }) +} + +// A positional argument that is not a subcommand used to be ignored, leaving +// an operator who mistyped a command watching a server start instead. +func TestRecoveryRejectsAnUnknownSubcommand(t *testing.T) { + var stdout, stderr bytes.Buffer + code := runRecovery([]string{"backp", "-db", "x"}, &stdout, &stderr) + if code == 0 { + t.Fatal("an unknown subcommand exited 0") + } + if !strings.Contains(stderr.String(), "backp") { + t.Errorf("refusal must name the unknown subcommand; stderr = %q", stderr.String()) + } +} + +// seedResource writes one revision of a control-plane resource through the +// store, the way the control plane's replace does. The request ID is derived +// from the label and the revision it produces: the store deduplicates a +// repeated request ID, so a fixed one would make the second write of a series +// a silent no-op. +func seedResource(t *testing.T, st *store.Store, kind, value, label string) { + t.Helper() + current, err := st.Resource(t.Context(), kind) + expected := int64(0) + if err == nil { + expected = current.Version + } + var document any + if err := json.Unmarshal([]byte(value), &document); err != nil { + t.Fatal(err) + } + encoded, err := json.Marshal(document) + if err != nil { + t.Fatal(err) + } + if _, err := st.PutResource(t.Context(), store.ResourceWrite{ + Kind: kind, Value: encoded, Actor: label, Source: "test", + RequestID: fmt.Sprintf("%s-%d", label, expected+1), ExpectedVersion: expected, + }); err != nil { + t.Fatalf("seed resource %s: %v", kind, err) + } +} + +// execRaw runs one statement against the database file the way an operator +// editing a broken store would, bypassing every check the store applies. +func execRaw(t *testing.T, path, statement string) error { + t.Helper() + db, err := sql.Open("sqlite", path) + if err != nil { + return err + } + defer func() { _ = db.Close() }() + _, err = db.ExecContext(t.Context(), statement) + return err +} diff --git a/docs/architecture/safe-change-and-recovery.md b/docs/architecture/safe-change-and-recovery.md index 564e27e00..6f10f70ff 100644 --- a/docs/architecture/safe-change-and-recovery.md +++ b/docs/architecture/safe-change-and-recovery.md @@ -120,21 +120,40 @@ file values would restore the second source of truth the control plane exists to remove. The cost is that a bad stored value stops the daemon starting, and the value can only be corrected through the daemon's own API. -### Recovery: not yet built +### Recovery: offline commands on the State Store -There is no escape hatch. Nothing skips the control plane at boot, and the only -writer for a resource is `ControlPlaneService.Command` on a running State Store. -An operator whose stored settings will not validate has no supported path back -short of editing `archie.db` by hand. +Nothing skips the control plane at boot, and the only writer for a resource is +`ControlPlaneService.Command` on a running State Store -- which is exactly what +the daemon fails closed without. `archie-state-store` therefore carries the +offline recovery commands (`archie-core-6zw0`). They work on the task database +file directly -- the configured `db_path` with `-tasks.sqlite` appended, not +`db_path` itself -- and need neither `archied` nor the Web UI. -`archie-core-6zw0` is the missing piece: offline commands that back up, restore, -validate and roll back the database without `archied` or the Web UI. Until it -lands, treat a settings change that requires a restart as a change that can -prevent one, and take a copy of `archie.db` first. +| Command | What it does | State Store | +| --- | --- | --- | +| `backup -db F -out F` | writes a transactionally consistent snapshot (`VACUUM INTO`), verified as a usable store before it is published | may be running | +| `restore -db F -from F` | replaces the database with a snapshot and drops the replaced file's `-wal`/`-shm` | must be stopped | +| `validate -db F` | checks the file the way boot meets it: readable, not corrupt, schema no newer than the binary, and every stored resource decoding under the definition that owns it | must be stopped | +| `rollback -db F -kind K [-revision N]` | replays an earlier revision of a stored resource through the ordinary replace, recording the rollback as a new revision | must be stopped | -Resource history is the in-band remedy and does exist: every write records its -prior value, so an edit made through the dashboard or a channel can be restored -by replaying an earlier revision while the State Store is up. +No rollback RPC exists and none is needed: a revision carries its own value, so +restoring one is the same replace the dashboard and the channels already use. +The offline command is that operation with the store stopped -- the one state in +which the daemon's own API cannot serve it. + +"Must be stopped" is checked rather than assumed. The serving process takes an +exclusive flock on `.lock` for its whole life, and `restore` and `rollback` +refuse with `task database is owned by another process` while another process +holds it. `backup` is the deliberate exception: the update installer runs inside +the process it is replacing and cannot stop the State Store, so the snapshot +that makes a failed update reversible is taken against a serving store -- and +`scripts/archie-update-install` takes it through this subcommand rather than +through its own `sqlite3` shell, so the installer and an operator run one +implementation. + +`rollback` is the surgical route back from a bad stored value; `restore` is the +blunt one, and is also the way back from a schema migration a release cannot +grow out of. ## Gateway restart: two constraints learned the hard way diff --git a/internal/app/archied/state_store.go b/internal/app/archied/state_store.go index d2a1da380..b1b919d35 100644 --- a/internal/app/archied/state_store.go +++ b/internal/app/archied/state_store.go @@ -215,7 +215,25 @@ func (b *boot) openStateStore(ctx context.Context) error { log.Error("configure bindings cipher", "err", err) return err } - st, err := openProductionTaskStore(ctx, taskDBPath(cfg.DBPath), store.WithBindingCipher(bindingCipher)) + // Claim the database before opening it. This process is the file's owner + // for as long as it lives, and the claim is what an offline recovery + // command checks before it rewrites the file: without it, "the State Store + // is stopped" is a rule the operator is trusted to have followed rather + // than a fact the command can verify. A second State Store on the same + // database fails here, which is the single-owner invariant stated instead + // of merely documented. + path := taskDBPath(cfg.DBPath) + ownership, err := store.AcquireOwnership(path) + if err != nil { + log.Error("claim state store ownership", "err", err) + return err + } + b.addCleanup(func() { + if err := ownership.Release(); err != nil { + log.Error("release state store ownership", "err", err) + } + }) + st, err := openProductionTaskStore(ctx, path, store.WithBindingCipher(bindingCipher)) if err != nil { log.Error("open state store", "err", err) return err diff --git a/internal/app/archied/state_store_recovery.go b/internal/app/archied/state_store_recovery.go new file mode 100644 index 000000000..bf0acc93d --- /dev/null +++ b/internal/app/archied/state_store_recovery.go @@ -0,0 +1,210 @@ +// state_store_recovery.go composes the offline recovery subcommands of the +// standalone archie-state-store process. They exist because the control plane +// fails closed: a stored setting that will not validate stops archied +// starting, and the in-band remedy -- replaying an earlier revision while the +// State Store is up -- is unavailable in exactly the case that needs it +// (docs/architecture/safe-change-and-recovery.md). +// +// Every operation works on the task database file directly, with the State +// Store stopped, and the ones that write refuse when another process owns the +// file. Backup is the exception and deliberately so: the update installer +// cannot stop the process that runs it, so the snapshot that makes a failed +// update reversible is taken against a serving store through SQLite's own +// consistent copy. +package archied + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/samcharles93/archie-core/internal/app/controlplane" + pb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1" + "github.com/samcharles93/archie-core/internal/store" +) + +// The audit identity every offline rollback records. A rollback is one more +// revision of the resource, written through the ordinary replace, so the +// operator can see who went back and when -- and can go forward again the same +// way, with the State Store up or down. +const ( + OfflineRollbackSource = "archie-state-store rollback" + OfflineRollbackActor = "operator:offline-recovery" +) + +// Recovery operations, as named on the command line. +const ( + RecoveryBackup = "backup" + RecoveryRestore = "restore" + RecoveryValidate = "validate" + RecoveryRollback = "rollback" +) + +// StateStoreRecoveryOptions are the process inputs for one offline recovery +// operation. +type StateStoreRecoveryOptions struct { + // Operation is one of backup, restore, validate, rollback. + Operation string + // DB is the task database file the State Store owns: the configured + // db_path with "-tasks.sqlite" appended, not the configured path itself. + DB string + // Out is the snapshot backup writes. + Out string + // From is the snapshot restore reads. + From string + // Kind is the control-plane resource kind rollback replays. + Kind string + // Revision is the revision rollback replays. Zero means the newest + // revision older than the one the resource carries now. + Revision int64 +} + +// RunStateStoreRecovery performs one offline operation on the task database and +// returns the line the command reports on stdout. +func RunStateStoreRecovery(ctx context.Context, options StateStoreRecoveryOptions) (string, error) { + switch options.Operation { + case RecoveryBackup: + if options.DB == "" || options.Out == "" { + return "", errors.New("backup requires -db and -out") + } + if err := store.Backup(ctx, options.DB, options.Out); err != nil { + return "", err + } + return fmt.Sprintf("backed up %s to %s", options.DB, options.Out), nil + case RecoveryRestore: + if options.DB == "" || options.From == "" { + return "", errors.New("restore requires -db and -from") + } + if err := store.Restore(ctx, options.DB, options.From); err != nil { + return "", err + } + return fmt.Sprintf("restored %s from %s; start the State Store again", options.DB, options.From), nil + case RecoveryValidate: + if options.DB == "" { + return "", errors.New("validate requires -db") + } + return validateStore(ctx, options.DB) + case RecoveryRollback: + if options.DB == "" || options.Kind == "" { + return "", errors.New("rollback requires -db and -kind") + } + return rollbackResource(ctx, options) + default: + return "", fmt.Errorf("unknown recovery command %q", options.Operation) + } +} + +// validateStore answers "would archied start against this file". The file-level +// checks come first, so the store is only read once its schema is one this +// binary understands; the resource check then decodes every stored value with +// the definition that owns it, which is the same validation a write applies. +func validateStore(ctx context.Context, path string) (string, error) { + version, err := store.ValidateFile(ctx, path) + if err != nil { + return "", err + } + st, err := store.OpenReadOnly(ctx, path) + if err != nil { + return "", err + } + defer func() { _ = st.Close() }() + // The resources table is only absent from a store written before the control + // plane existed. That store holds no settings to validate, and the serving + // process creates the table on its next start -- refusing it here would send + // an operator to restore a snapshot for nothing. + checked := 0 + stored, err := st.StoresResources(ctx) + if err != nil { + return "", err + } + if stored { + checked, err = controlplane.NewServer(st).ValidateStored(ctx) + if err != nil { + return "", err + } + } + return fmt.Sprintf("%s is a valid store at schema version %d; %d stored resources validate", path, version, checked), nil +} + +// rollbackResource is the one operation the Web UI cannot be asked to perform +// here: the daemon fails closed on the bad value, so the store that would serve +// the rollback is the store that will not start. It replays the revision's own +// value through the ordinary replace, so no rollback RPC exists or is needed, +// and the rollback is audited as one more revision. +func rollbackResource(ctx context.Context, options StateStoreRecoveryOptions) (string, error) { + // Before the lock and before the open: opening a store creates the file, + // and a rollback pointed at a mistyped path must not leave an empty + // database where the operator expected theirs. + if err := store.RequireDatabase(options.DB); err != nil { + return "", err + } + ownership, err := store.AcquireOwnership(options.DB) + if err != nil { + return "", err + } + defer func() { _ = ownership.Release() }() + + st, err := store.Open(ctx, options.DB) + if err != nil { + return "", err + } + defer func() { _ = st.Close() }() + + server := controlplane.NewServer(st) + if !server.Owns(options.Kind) { + return "", fmt.Errorf("unknown resource kind %q", options.Kind) + } + current, err := st.Resource(ctx, options.Kind) + if errors.Is(err, store.ErrResourceNotFound) { + return "", fmt.Errorf("%s has no stored value to roll back", options.Kind) + } + if err != nil { + return "", fmt.Errorf("read %s: %w", options.Kind, err) + } + revision, err := revisionToReplay(ctx, st, options.Kind, current.Version, options.Revision) + if err != nil { + return "", err + } + replaced, err := server.Command(ctx, &pb.CommandRequest{ + Kind: options.Kind, + Command: "replace", + ValueJson: revision.Value, + ExpectedVersion: current.Version, + Actor: OfflineRollbackActor, + Source: OfflineRollbackSource, + RequestId: fmt.Sprintf("recovery-rollback-%s-%d", options.Kind, time.Now().UTC().UnixNano()), + }) + if err != nil { + return "", fmt.Errorf("replay %s at revision %d: %w", options.Kind, revision.Version, err) + } + return fmt.Sprintf("rolled back %s from version %d to the value recorded at version %d; the store now holds version %d", + options.Kind, current.Version, revision.Version, replaced.GetResource().GetVersion()), nil +} + +// revisionToReplay picks the value to put back: the revision the operator +// named, or the newest one older than the value the resource carries now. +func revisionToReplay(ctx context.Context, st *store.Store, kind string, current, requested int64) (store.Resource, error) { + history, err := st.ResourceHistory(ctx, kind, 0) + if err != nil { + return store.Resource{}, fmt.Errorf("read %s history: %w", kind, err) + } + if requested > 0 { + for _, revision := range history { + if revision.Version != requested { + continue + } + if revision.Version == current { + return store.Resource{}, fmt.Errorf("%s revision %d is the current version; there is nothing to roll back to", kind, requested) + } + return revision, nil + } + return store.Resource{}, fmt.Errorf("%s has no revision %d", kind, requested) + } + for _, revision := range history { + if revision.Version < current { + return revision, nil + } + } + return store.Resource{}, fmt.Errorf("%s has no earlier revision to roll back to", kind) +} diff --git a/internal/app/controlplane/registry.go b/internal/app/controlplane/registry.go index b49186ce9..ddf66142a 100644 --- a/internal/app/controlplane/registry.go +++ b/internal/app/controlplane/registry.go @@ -89,6 +89,42 @@ func (s *Server) ImportConfig(ctx context.Context, cfg config.Config) (map[strin return versions, nil } +// Owns reports whether kind is a resource the control plane owns, so a caller +// acting on a stored value can refuse a kind nobody defines before it reaches +// the store. +func (s *Server) Owns(kind string) bool { + _, ok := s.definitions[kind] + return ok +} + +// ValidateStored decodes every stored resource with the definition that owns +// it, using the same Decode the write path applies, and reports how many +// resources it checked. Errors name the kind and the revision it was stored at, +// which is what an operator needs to roll the value back. +// +// This is how an offline validate reproduces the refusal that stops archied +// starting: the check is the writer's own, so it cannot drift from what a +// replace would accept, and a value can only be stored unvalidated by bypassing +// the write path entirely. +func (s *Server) ValidateStored(ctx context.Context) (int, error) { + checked := 0 + var failures []error + for _, definition := range s.ordered { + resource, err := s.store.Resource(ctx, definition.Kind) + if errors.Is(err, store.ErrResourceNotFound) { + continue + } + if err != nil { + return checked, fmt.Errorf("read %s: %w", definition.Kind, err) + } + checked++ + if _, err := definition.Decode(resource.Value); err != nil { + failures = append(failures, fmt.Errorf("%s at version %d: %w", definition.Kind, resource.Version, err)) + } + } + return checked, errors.Join(failures...) +} + func validateAs[T any](input []byte, validate func(T) error) error { var value T decoder := json.NewDecoder(bytes.NewReader(input)) diff --git a/internal/releaseupdate/install_script_test.go b/internal/releaseupdate/install_script_test.go index fdae8fac4..77f58e27e 100644 --- a/internal/releaseupdate/install_script_test.go +++ b/internal/releaseupdate/install_script_test.go @@ -382,6 +382,13 @@ esac `) writeFakeCommand(t, fakeDir, "go", ` printf '%s\n' "go $*" >> "$ARCHIE_TEST_CALLS" +# The pre-flight task-database backup runs through the release's own +# archie-state-store subcommand, so a release whose recovery path cannot run +# must refuse the update. +if [ "$1" = "run" ]; then + [ "${ARCHIE_TEST_GO_FAILS_RUN:-}" = "1" ] && exit 1 + exit 0 +fi out="" while [ "$#" -gt 0 ]; do if [ "$1" = "-o" ]; then out="$2"; break; fi @@ -745,6 +752,10 @@ func TestUpdateWatchdogCyclesEveryUnitInDependencyOrder(t *testing.T) { // value up produces an empty backup, and a rollback then restores that empty // file over the database and deletes its -wal/-shm. A rollback that destroys // the task store is worse than no rollback. +// +// The snapshot is taken by the release's own recovery subcommand rather than +// by hand-rolled shell, so the installer and an operator run the same code and +// the two cannot drift apart. func TestUpdateInstallBacksUpTheSiblingTaskDatabase(t *testing.T) { work := t.TempDir() configured := filepath.Join(work, "archie.db") @@ -764,7 +775,8 @@ func TestUpdateInstallBacksUpTheSiblingTaskDatabase(t *testing.T) { "ARCHIE_TASK_DB_BACKUP": backup, }, "db_path = \""+configured+"\"\n\n[containers]\nimage = 'registry.example/archie-agent:stable'\n", false) - assertCallContains(t, calls, "sqlite3", realStore, ".backup") + assertCallContains(t, calls, "go run ./cmd/archie-state-store backup", "-db "+realStore, "-out "+backup) + assertCallAbsent(t, calls, "sqlite3") assertCallContains(t, calls, "systemd-run", "--setenv=ARCHIE_TASK_DB_PATH="+realStore) // The zero-byte placeholder is not the store, and must never be the thing // handed to the watchdog as ARCHIE_TASK_DB_PATH. @@ -793,9 +805,9 @@ func TestUpdateInstallIgnoresDbPathFromOtherSections(t *testing.T) { "ARCHIE_TASK_DB_BACKUP": backup, }, "db_path = \""+configured+"\"\n\n[indexing]\ndb_path = \""+indexStore+"\"\n\n[containers]\nimage = 'registry.example/archie-agent:stable'\n", false) - assertCallContains(t, calls, "sqlite3", realStore) + assertCallContains(t, calls, "go run ./cmd/archie-state-store backup", "-db "+realStore) for _, call := range calls { - if strings.Contains(call, "sqlite3") && strings.Contains(call, indexStore) { + if strings.Contains(call, "go run ./cmd/archie-state-store backup") && strings.Contains(call, indexStore) { t.Errorf("backed up the indexing database %s: %q", indexStore, call) } } @@ -815,7 +827,7 @@ func TestUpdateInstallRefusesWhenTaskDatabaseCannotBeBackedUp(t *testing.T) { "ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0", "ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0", "ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0", - "ARCHIE_TEST_SQLITE3_FAILS": "1", + "ARCHIE_TEST_GO_FAILS_RUN": "1", }, "db_path = \""+configured+"\"\n\n[containers]\nimage = 'registry.example/archie-agent:stable'\n", true) if !strings.Contains(output, "task database") { diff --git a/internal/store/maintenance.go b/internal/store/maintenance.go new file mode 100644 index 000000000..12c3af029 --- /dev/null +++ b/internal/store/maintenance.go @@ -0,0 +1,253 @@ +package store + +import ( + "context" + "database/sql" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" +) + +// This file is the offline half of the store's contract with its owner: the +// operations the State Store process cannot perform on itself, because they +// read or replace the very file it has open. They are what +// docs/prds/runtime-control-plane.md ("Bootstrap, migration, and recovery") +// asks for -- back up, restore and validate the database without archied or +// the Web UI -- and every one of them works on the file directly. +// +// The read-only operations never take the ownership lock, because they are +// what an operator runs against a serving store. The one that replaces the +// file does, because a rewrite under a running server is the only way an +// offline command can destroy a task store. + +// OpenReadOnly opens an existing store for reading only: no schema is created +// and no migration runs. That matters for recovery -- the snapshot an update +// takes has to stay exactly what the release that wrote it expects, and +// migrating a copy during verification would hand a rollback a schema its +// binary refuses. +func OpenReadOnly(ctx context.Context, path string) (*Store, error) { + if err := RequireDatabase(path); err != nil { + return nil, err + } + db, err := sql.Open("sqlite", readonlyDSN(path)) + if err != nil { + return nil, err + } + db.SetMaxOpenConns(1) + // sql.Open is lazy: the first statement is where a file that is not a + // database, a missing WAL sidecar or an unreadable path surfaces. + var tables int + if err := db.QueryRowContext(ctx, `SELECT count(*) FROM sqlite_master`).Scan(&tables); err != nil { + return nil, errors.Join(fmt.Errorf("read task database %s: %w", path, err), db.Close()) + } + return &Store{db: db}, nil +} + +// readonlyDSN reads the store without the write-ahead log pragma the serving +// process sets: a reader has no business changing the database's journal mode, +// and the read-only flag makes an accidental write an error rather than a +// silent modification of a file someone else owns. +func readonlyDSN(path string) string { + separator := "?" + if strings.HasSuffix(path, "?") || strings.HasSuffix(path, "&") { + separator = "" + } else if strings.Contains(path, "?") { + separator = "&" + } + return path + separator + "mode=ro&_pragma=busy_timeout(5000)" +} + +// StoresResources reports whether the store's file carries the control-plane +// resources table. A store written before the control plane existed has none +// and holds no stored settings to validate; the serving process creates the +// table on its next start, so an offline check must tell that apart from +// resources it cannot read. +func (s *Store) StoresResources(ctx context.Context) (bool, error) { + var tables int + if err := s.db.QueryRowContext(ctx, `SELECT count(*) FROM sqlite_master WHERE type='table' AND name='resources'`).Scan(&tables); err != nil { + return false, fmt.Errorf("inspect store schema: %w", err) + } + return tables > 0, nil +} + +// RequireDatabase reports whether path is an existing store file. A writer +// that must not create the file it fails to find asks this first: a recovery +// command pointed at a mistyped path has to refuse, not leave a fresh empty +// database where an operator expected theirs. +func RequireDatabase(path string) error { + if _, err := os.Stat(path); err != nil { + return fmt.Errorf("task database %s: %w", path, err) + } + return nil +} + +// ValidateFile checks an existing store the way the serving process meets it +// on startup and reports the schema version it carries: the file is a SQLite +// database this binary can read, it is not corrupt, and its schema is not +// newer than this binary supports. +func ValidateFile(ctx context.Context, path string) (int, error) { + st, err := OpenReadOnly(ctx, path) + if err != nil { + return 0, err + } + defer func() { _ = st.Close() }() + return st.check(ctx) +} + +// check runs the file-level checks. integrity_check returns one row per +// problem, or the single row "ok"; reading the first row is therefore the +// verdict rather than a sample. +func (s *Store) check(ctx context.Context) (int, error) { + var result string + if err := s.db.QueryRowContext(ctx, `PRAGMA integrity_check`).Scan(&result); err != nil { + return 0, fmt.Errorf("integrity check: %w", err) + } + if result != "ok" { + return 0, fmt.Errorf("integrity check: %s", result) + } + var version int + if err := s.db.QueryRowContext(ctx, `PRAGMA user_version`).Scan(&version); err != nil { + return 0, fmt.Errorf("read schema version: %w", err) + } + if version > taskSchemaVersion { + return 0, fmt.Errorf("database schema version %d is newer than supported version %d", version, taskSchemaVersion) + } + return version, nil +} + +// Backup writes a transactionally consistent snapshot of database to snapshot, +// using SQLite's own copy, so a store that is being written to cannot produce +// a torn file. +// +// It deliberately does not take the ownership lock. The update installer runs +// inside the process it is updating and cannot stop the State Store, so the +// snapshot that makes a failed update reversible is taken against a serving +// store; a backup that insisted on an exclusively owned file would break the +// one caller that exists. The snapshot is written beside the destination and +// renamed into place, so an interrupted backup never replaces a snapshot that +// was good with one that is not. +func Backup(ctx context.Context, database, snapshot string) error { + if err := RequireDatabase(database); err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(snapshot), 0o755); err != nil { + return fmt.Errorf("create snapshot directory: %w", err) + } + db, err := sql.Open("sqlite", sqliteDSN(database)) + if err != nil { + return fmt.Errorf("open task database %s: %w", database, err) + } + defer func() { _ = db.Close() }() + + staging := snapshot + ".tmp" + if err := os.Remove(staging); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("clear staging snapshot %s: %w", staging, err) + } + if _, err := db.ExecContext(ctx, `VACUUM INTO ?`, staging); err != nil { + return errors.Join(fmt.Errorf("snapshot %s: %w", database, err), os.Remove(staging)) + } + // A snapshot that is not itself a usable store is worse than no snapshot: + // the update path treats its existence as proof that a rollback is + // possible. + if _, err := ValidateFile(ctx, staging); err != nil { + return errors.Join(fmt.Errorf("snapshot is not a usable store: %w", err), os.Remove(staging)) + } + if err := os.Rename(staging, snapshot); err != nil { + return errors.Join(fmt.Errorf("publish snapshot %s: %w", snapshot, err), os.Remove(staging)) + } + return nil +} + +// Restore replaces database with the snapshot at snapshot. The snapshot is +// verified before anything is destroyed, so a file that is not a usable store +// cannot take the database down with it, and the database is replaced by a +// rename, so a failure leaves either the old file or the new one rather than a +// mixture. +// +// It holds the ownership lock for its duration: rewriting a database a running +// State Store still has open would leave that process writing to an unlinked +// inode while the store served a file nobody owns. +func Restore(ctx context.Context, database, snapshot string) error { + if _, err := os.Stat(filepath.Dir(database)); err != nil { + return fmt.Errorf("restore destination %s: %w", database, err) + } + if same, err := sameFile(database, snapshot); err != nil { + return err + } else if same { + return fmt.Errorf("restore snapshot %s is the database itself", snapshot) + } + ownership, err := AcquireOwnership(database) + if err != nil { + return err + } + defer func() { _ = ownership.Release() }() + + if _, err := ValidateFile(ctx, snapshot); err != nil { + return fmt.Errorf("snapshot is not a usable store: %w", err) + } + staging := database + ".restore.tmp" + if err := copyFile(snapshot, staging); err != nil { + return errors.Join(err, os.Remove(staging)) + } // The replaced database's write-ahead log belongs to the file that is + // going away. It goes first: between the two calls the database is merely + // incomplete, whereas a WAL left beside the file it does not describe is + // the state SQLite cannot recover from. + for _, sidecar := range []string{"-wal", "-shm"} { + if err := os.Remove(database + sidecar); err != nil && !os.IsNotExist(err) { + return errors.Join(fmt.Errorf("remove %s%s: %w", database, sidecar, err), os.Remove(staging)) + } + } + if err := os.Rename(staging, database); err != nil { + return errors.Join(fmt.Errorf("publish restored database %s: %w", database, err), os.Remove(staging)) + } + return nil +} + +func sameFile(a, b string) (bool, error) { + first, err := os.Stat(a) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + second, err := os.Stat(b) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + return os.SameFile(first, second), nil +} + +// copyFile writes source over a new file at path, flushed to disk before it is +// returned: the rename that publishes it must not reach the directory entries +// ahead of the bytes they point at. The snapshot's own permissions are kept, so +// a restore does not silently change who may read the operator's database. +func copyFile(source, path string) (retErr error) { + info, err := os.Stat(source) + if err != nil { + return fmt.Errorf("stat snapshot %s: %w", source, err) + } + in, err := os.Open(source) + if err != nil { + return fmt.Errorf("open snapshot %s: %w", source, err) + } + defer func() { retErr = errors.Join(retErr, in.Close()) }() + out, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, info.Mode().Perm()) + if err != nil { + return fmt.Errorf("create %s: %w", path, err) + } + defer func() { retErr = errors.Join(retErr, out.Close()) }() + if _, err := io.Copy(out, in); err != nil { + return fmt.Errorf("write %s: %w", path, err) + } + if err := out.Sync(); err != nil { + return fmt.Errorf("sync %s: %w", path, err) + } + return nil +} diff --git a/internal/store/maintenance_test.go b/internal/store/maintenance_test.go new file mode 100644 index 000000000..95eda0c86 --- /dev/null +++ b/internal/store/maintenance_test.go @@ -0,0 +1,92 @@ +package store + +import ( + "context" + "database/sql" + "errors" + "fmt" + "os" + "path/filepath" + "testing" +) + +// Ownership is what makes "the State Store is stopped" a fact an offline +// command can check, so the primitive itself is asserted here rather than only +// through the commands that use it: exclusivity, release, and per-database +// independence. +func TestOwnershipIsExclusivePerDatabase(t *testing.T) { + dir := t.TempDir() + first := filepath.Join(dir, "archie.db-tasks.sqlite") + second := filepath.Join(dir, "other.db-tasks.sqlite") + + held, err := AcquireOwnership(first) + if err != nil { + t.Fatalf("first AcquireOwnership: %v", err) + } + if _, err := AcquireOwnership(first); !errors.Is(err, ErrStoreOwned) { + t.Fatalf("second AcquireOwnership on the same database = %v, want ErrStoreOwned", err) + } + // A different database is a different claim: one process owning its store + // must not stop another from owning its own. + elsewhere, err := AcquireOwnership(second) + if err != nil { + t.Fatalf("AcquireOwnership on an unrelated database: %v", err) + } + if _, err := os.Stat(first + ".lock"); err != nil { + t.Fatalf("ownership left no sidecar to lock: %v", err) + } + + if err := held.Release(); err != nil { + t.Fatalf("Release: %v", err) + } + reacquired, err := AcquireOwnership(first) + if err != nil { + t.Fatalf("AcquireOwnership after Release: %v", err) + } + if err := errors.Join(reacquired.Release(), elsewhere.Release()); err != nil { + t.Fatalf("release on cleanup: %v", err) + } +} + +// The snapshot an update takes is the state a rollback restores, and it is +// restored by the release that wrote it. Verifying the copy must therefore not +// migrate it: a snapshot silently upgraded to this binary's schema would be +// refused by the older binary it exists to serve. +func TestBackupSnapshotKeepsTheSchemaVersionItCopied(t *testing.T) { + dir := t.TempDir() + database := filepath.Join(dir, "archie.db-tasks.sqlite") + snapshot := filepath.Join(dir, "snapshot.sqlite") + st, err := Open(t.Context(), database) + if err != nil { + t.Fatal(err) + } + if err := st.Close(); err != nil { + t.Fatal(err) + } + // An older release's file: the schema version it wrote, which this binary + // is expected to migrate when the store is next opened for serving. + if err := setUserVersion(t.Context(), database, taskSchemaVersion-1); err != nil { + t.Fatal(err) + } + + if err := Backup(t.Context(), database, snapshot); err != nil { + t.Fatalf("Backup: %v", err) + } + version, err := ValidateFile(t.Context(), snapshot) + if err != nil { + t.Fatalf("ValidateFile on the snapshot: %v", err) + } + if version != taskSchemaVersion-1 { + t.Fatalf("snapshot schema version = %d, want the %d it copied", version, taskSchemaVersion-1) + } +} + +func setUserVersion(ctx context.Context, path string, version int) error { + db, err := sql.Open("sqlite", path) + if err != nil { + return err + } + defer func() { _ = db.Close() }() + _, err = db.ExecContext(ctx, fmt.Sprintf(`PRAGMA user_version = %d`, version)) + return err +} diff --git a/internal/store/ownership.go b/internal/store/ownership.go new file mode 100644 index 000000000..b0743d89f --- /dev/null +++ b/internal/store/ownership.go @@ -0,0 +1,68 @@ +package store + +import ( + "errors" + "fmt" + "os" + "path/filepath" +) + +// ErrStoreOwned is returned when a task database is already owned by another +// process. Ownership is what makes "the State Store is stopped" a fact an +// offline command can check rather than a rule the operator is trusted to +// follow: overwriting a database a running server still holds leaves it +// writing to an unlinked inode while the store serves a file nobody owns. +var ErrStoreOwned = errors.New("task database is owned by another process") + +// Ownership is one process's exclusive claim over one task database, held for +// as long as that process owns the file. The serving State Store takes it for +// its whole life; the offline recovery commands take it for the duration of an +// operation that rewrites the file. +// +// The claim lives in a sidecar file beside the database rather than on the +// database inode itself, so it survives the rename a restore performs: after +// the swap the sidecar is still the one path every process locks. +type Ownership struct { + f *os.File +} + +// AcquireOwnership takes the store's ownership lock without blocking. A +// database another process owns comes back as ErrStoreOwned, never as a wait: +// a process that must not proceed has to hear so, and a blocking lock would +// turn a mistyped command into a hung terminal. +func AcquireOwnership(path string) (*Ownership, error) { + lockPath := ownershipLockPath(path) + if err := os.MkdirAll(filepath.Dir(lockPath), 0o755); err != nil { + return nil, fmt.Errorf("create store directory for %s: %w", lockPath, err) + } + f, err := os.OpenFile(lockPath, os.O_RDWR|os.O_CREATE, 0o644) + if err != nil { + return nil, fmt.Errorf("open store lock %s: %w", lockPath, err) + } + taken, err := lockFile(f) + if err != nil { + return nil, errors.Join(fmt.Errorf("lock %s: %w", lockPath, err), f.Close()) + } + if !taken { + return nil, errors.Join(fmt.Errorf("%s: %w", path, ErrStoreOwned), f.Close()) + } + return &Ownership{f: f}, nil +} + +// Release drops the claim and closes the sidecar. Closing the descriptor alone +// would release it too; the explicit unlock keeps the intent readable and +// makes Release idempotent. +func (o *Ownership) Release() error { + if o == nil || o.f == nil { + return nil + } + err := unlockFile(o.f) + f := o.f + o.f = nil + return errors.Join(err, f.Close()) +} + +// ownershipLockPath is the sidecar every owner of path locks. It is derived +// from the database path, so the State Store process and an offline command +// that were given the same database agree on it without configuration. +func ownershipLockPath(path string) string { return path + ".lock" } diff --git a/internal/store/ownership_unix.go b/internal/store/ownership_unix.go new file mode 100644 index 000000000..3d0d2bb4b --- /dev/null +++ b/internal/store/ownership_unix.go @@ -0,0 +1,30 @@ +//go:build unix + +package store + +import ( + "errors" + "os" + "syscall" +) + +// lockFile takes a non-blocking exclusive flock on f. The boolean reports +// whether it was taken: a lock another process holds (EWOULDBLOCK) is a normal +// answer to "is the State Store running?", not an error. +func lockFile(f *os.File) (bool, error) { + err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) + if errors.Is(err, syscall.EWOULDBLOCK) { + return false, nil + } + if err != nil { + return false, err + } + return true, nil +} + +func unlockFile(f *os.File) error { + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_UN); err != nil { + return err + } + return nil +} diff --git a/internal/store/ownership_windows.go b/internal/store/ownership_windows.go new file mode 100644 index 000000000..e282ae29d --- /dev/null +++ b/internal/store/ownership_windows.go @@ -0,0 +1,23 @@ +//go:build windows + +// flock is a POSIX primitive and Windows is not a supported deployment surface +// (CLAUDE.md's deployment model is Linux and macOS), so this file fails closed: +// every lock attempt reports an error rather than silently degrading to "no +// lock". An offline command that cannot prove it owns the database must refuse +// to rewrite it, which is the only safe reading of an unavailable primitive. +// +// If a Windows deployment ever becomes supported, swap this for a real +// LockFileEx/UnlockFileEx pair; nothing else in the package needs to change. + +package store + +import ( + "errors" + "os" +) + +func lockFile(*os.File) (bool, error) { + return false, errors.New("store ownership locking requires a POSIX flock, which this platform does not provide") +} + +func unlockFile(*os.File) error { return nil } diff --git a/scripts/archie-update-install b/scripts/archie-update-install index ce3635f63..30c43bef4 100755 --- a/scripts/archie-update-install +++ b/scripts/archie-update-install @@ -183,10 +183,18 @@ fi # that cannot take one must not start: once binaries are replaced, a failure # leaves the previous release unable to read the store it just migrated. This # runs ahead of the adapter refresh below so a refusal changes nothing at all. +# +# The snapshot is taken by the recovery subcommand of the release being +# installed, from its own checkout, rather than by hand-rolled sqlite3 shell: +# the operator running `archie-state-store backup` by hand and the installer +# are then the same implementation, and the two cannot drift. The installed +# binary is the previous release, which may predate the subcommand, so it +# cannot be the one asked -- and a release whose recovery path will not build is a +# release that cannot back up, which is exactly what this pre-flight refuses. if [[ ",$components," == *,daemon,* ]] && [ -n "$TASK_DB_PATH" ] && [ -f "$TASK_DB_PATH" ]; then echo "==> backing up task database for schema-safe rollback" mkdir -p "$(dirname "$TASK_DB_BACKUP")" - if ! sqlite3 "$TASK_DB_PATH" ".backup '$TASK_DB_BACKUP'"; then + if ! go run ./cmd/archie-state-store backup -db "$TASK_DB_PATH" -out "$TASK_DB_BACKUP"; then echo "refusing to install $GW: could not back up the task database at $TASK_DB_PATH, so a failed update could not be rolled back" >&2 exit 1 fi