diff --git a/cmd/archie-state-store/main.go b/cmd/archie-state-store/main.go index 130956aad..6ca5ba8a5 100644 --- a/cmd/archie-state-store/main.go +++ b/cmd/archie-state-store/main.go @@ -15,6 +15,7 @@ import ( "os" "os/signal" "path/filepath" + "strings" "syscall" "github.com/samcharles93/archie-core/internal/app/archied" @@ -23,6 +24,13 @@ import ( func main() { os.Exit(run()) } func run() int { + // A leading non-flag argument selects a recovery subcommand; anything else, + // including no arguments at all, serves the State Store contract. An + // unrecognised word is refused rather than ignored, so a mistyped recovery + // command never leaves an operator watching a server start instead. + if args := os.Args[1:]; len(args) > 0 && !strings.HasPrefix(args[0], "-") { + return runRecovery(args, os.Stdout, os.Stderr) + } base, err := os.UserConfigDir() if err != nil { fmt.Fprintln(os.Stderr, err) @@ -34,6 +42,10 @@ func run() int { flag.StringVar(&options.Listen, "listen", "", "state store gRPC listen address (defaults to [services.state].listen, else 127.0.0.1:9090)") flag.StringVar(&options.Token, "token", "", "bearer token required for a non-loopback listener (defaults to [services.state].target_token)") flag.StringVar(&options.ReadyAddr, "ready-addr", "", "optional readiness HTTP listen address (e.g. 127.0.0.1:9091)") + flag.Usage = func() { + fmt.Fprintf(flag.CommandLine.Output(), "usage: archie-state-store [flags] # serve the State Store gRPC contract\n\noffline recovery: archie-state-store -h\n\n") + flag.PrintDefaults() + } flag.Parse() ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() diff --git a/cmd/archie-state-store/recovery.go b/cmd/archie-state-store/recovery.go new file mode 100644 index 000000000..259410236 --- /dev/null +++ b/cmd/archie-state-store/recovery.go @@ -0,0 +1,94 @@ +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + + "github.com/samcharles93/archie-core/internal/app/archied" +) + +// defaultConfigPath is the configuration the serve path reads by default, and +// the one the daemon boots with. It is the daemon's own rule rather than a +// second derivation of it: os.UserConfigDir (what the sibling processes use) +// disagrees with archied on darwin and rejects a relative XDG_CONFIG_HOME +// outright, which would leave validate answering about a file nobody boots. +func defaultConfigPath() string { return archied.DefaultConfigPath() } + +// recoveryUsage is the whole flag surface of the offline recovery commands. +// They are subcommands of this binary rather than a recovery binary of their +// own: offline database maintenance belongs to the process that owns the file +// and its path layout (docs/architecture/organisation.md#process-boundaries). +const recoveryUsage = `usage: archie-state-store [flags] + +With no command, archie-state-store serves the State Store gRPC contract. + +The recovery commands operate on the task database file directly -- the +configured db_path with "-tasks.sqlite" appended -- and are how an operator +recovers a database the control plane will not start on. Stop the State Store +first: restore and rollback rewrite the file and refuse while another process +owns it, while backup takes a consistent snapshot of a serving store. + + backup -db FILE -out FILE write a snapshot of the database + restore -db FILE -from FILE replace the database with a snapshot + validate -db FILE [-config FILE] check the file, its stored settings and the + configuration the daemon would boot with + rollback -db FILE -kind KIND replay an earlier revision of a stored + [-revision N] resource through the ordinary replace +` + +// runRecovery parses and performs one offline recovery command, reporting the +// operation's summary on stdout. Exit codes follow the serve path: 0 succeeded, +// 1 the operation failed, 2 the command line was wrong. +func runRecovery(args []string, stdout, stderr io.Writer) int { + if len(args) == 0 { + fmt.Fprint(stderr, recoveryUsage) + return 2 + } + command := args[0] + options := archied.StateStoreRecoveryOptions{Operation: command} + switch command { + case archied.RecoveryBackup, archied.RecoveryRestore, archied.RecoveryValidate, archied.RecoveryRollback: + default: + fmt.Fprintf(stderr, "archie-state-store: unknown command %q\n\n", command) + fmt.Fprint(stderr, recoveryUsage) + return 2 + } + + flags := flag.NewFlagSet("archie-state-store "+command, flag.ContinueOnError) + flags.SetOutput(stderr) + flags.Usage = func() { fmt.Fprint(stderr, recoveryUsage) } + flags.StringVar(&options.DB, "db", "", "task database file the State Store owns (-tasks.sqlite)") + flags.StringVar(&options.Out, "out", "", "snapshot file backup writes") + flags.StringVar(&options.From, "from", "", "snapshot file restore reads") + flags.StringVar(&options.Kind, "kind", "", "control-plane resource kind rollback replays") + flags.Int64Var(&options.Revision, "revision", 0, "revision rollback replays (default: the newest one older than the current value)") + // Only validate asks a question about the process rather than the file, so + // only validate takes the configuration the process would boot with. + if command == archied.RecoveryValidate { + flags.StringVar(&options.Config, "config", defaultConfigPath(), "configuration file or directory the daemon boots with") + flags.StringVar(&options.Overlay, "config-overlay", "", "configuration overlay file or directory") + } + if err := flags.Parse(args[1:]); err != nil { + // A requested help is not a wrong command line: the usage above is the + // answer, and the serve path exits 0 for the same request. + if errors.Is(err, flag.ErrHelp) { + return 0 + } + return 2 + } + if flags.NArg() > 0 { + fmt.Fprintf(stderr, "archie-state-store %s: unexpected argument %q\n", command, flags.Arg(0)) + return 2 + } + + summary, err := archied.RunStateStoreRecovery(context.Background(), options) + if err != nil { + fmt.Fprintf(stderr, "archie-state-store %s: %v\n", command, err) + return 1 + } + fmt.Fprintln(stdout, summary) + return 0 +} diff --git a/cmd/archie-state-store/recovery_test.go b/cmd/archie-state-store/recovery_test.go new file mode 100644 index 000000000..b58629f29 --- /dev/null +++ b/cmd/archie-state-store/recovery_test.go @@ -0,0 +1,770 @@ +// Package main tests the offline recovery subcommands of archie-state-store: +// backup, restore, validate and rollback against the task database file +// directly. They are the operator's only path back when the control plane's +// stored settings will not validate, because the daemon fails closed: the +// in-band remedy (replay an earlier revision while the State Store is up) is +// unavailable in exactly the case that needs it. +// +// The tests drive the subcommands the way an operator does -- through +// runRecovery, against real files on disk -- rather than through the internal +// helpers, so the flag surface and the exit codes are part of what is verified. + +package main + +import ( + "bytes" + "database/sql" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + + _ "modernc.org/sqlite" + + "github.com/samcharles93/archie-core/internal/app/archied" + "github.com/samcharles93/archie-core/internal/app/controlplane" + "github.com/samcharles93/archie-core/internal/infrastructure/configuration" + "github.com/samcharles93/archie-core/internal/infrastructure/workflowsteps" + "github.com/samcharles93/archie-core/internal/store" +) + +// runRecoveryCmd runs one subcommand the way main does and returns its exit +// code with both streams captured. +func runRecoveryCmd(t *testing.T, args ...string) (int, string, string) { + t.Helper() + var stdout, stderr bytes.Buffer + code := runRecovery(args, &stdout, &stderr) + return code, stdout.String(), stderr.String() +} + +// openTaskStore opens the task database the subcommands operate on. It does not +// take the ownership lock: only the serving process does that, and a test that +// wants a "live" store holds it explicitly with holdStoreOwnership. +func openTaskStore(t *testing.T, path string) *store.Store { + t.Helper() + st, err := store.Open(t.Context(), path) + if err != nil { + t.Fatalf("open task store %s: %v", path, err) + } + t.Cleanup(func() { _ = st.Close() }) + return st +} + +// holdStoreOwnership takes the ownership lock the running State Store holds for +// its whole life, so a test can prove a command refuses to touch a live store. +func holdStoreOwnership(t *testing.T, path string) { + t.Helper() + ownership, err := store.AcquireOwnership(path) + if err != nil { + t.Fatalf("acquire store ownership: %v", err) + } + t.Cleanup(func() { _ = ownership.Release() }) +} + +// seedTask writes one task so a snapshot can be distinguished from the live +// database by what it contains, not merely by its existence. +func seedTask(t *testing.T, st *store.Store, title string) { + t.Helper() + if _, err := st.EnqueueChatTask(t.Context(), "acme", "widget", title, "body", "implement", ""); err != nil { + t.Fatalf("seed task %q: %v", title, err) + } +} + +func taskTitles(t *testing.T, path string) []string { + t.Helper() + st := openTaskStore(t, path) + tasks, err := st.Tasks(t.Context(), 100) + if err != nil { + t.Fatalf("list tasks in %s: %v", path, err) + } + titles := make([]string, 0, len(tasks)) + for _, task := range tasks { + titles = append(titles, task.Title) + } + return titles +} + +// backupPath is the snapshot every restore test restores from. +func backupPath(t *testing.T, dir string) string { return filepath.Join(dir, "snapshot.sqlite") } + +// backup takes the snapshot the caller then restores, through the subcommand +// under test rather than through the store directly, so a broken backup fails +// the restore test too instead of silently passing an empty file. +func backup(t *testing.T, db, out string) { + t.Helper() + code, _, stderr := runRecoveryCmd(t, "backup", "-db", db, "-out", out) + if code != 0 { + t.Fatalf("backup exited %d: %s", code, stderr) + } +} + +// The update installer cannot stop the process that runs it, so it backs the +// task database up while the State Store is serving. VACUUM INTO is safe +// against a live writer; a backup that insisted on an exclusively-owned file +// would break the one caller it has, and the update would refuse to start. +func TestRecoveryBackupSnapshotsAStoreItsStateStoreIsServing(t *testing.T) { + dir := t.TempDir() + db := filepath.Join(dir, "archie.db-tasks.sqlite") + out := backupPath(t, dir) + seedTask(t, openTaskStore(t, db), "before") + holdStoreOwnership(t, db) + if err := os.WriteFile(out, []byte("stale snapshot"), 0o600); err != nil { + t.Fatal(err) + } + + code, stdout, stderr := runRecoveryCmd(t, "backup", "-db", db, "-out", out) + if code != 0 { + t.Fatalf("backup exited %d: %s", code, stderr) + } + if !strings.Contains(stdout, out) || !strings.Contains(stdout, db) { + t.Errorf("backup must report the database and the snapshot it wrote; stdout = %q", stdout) + } + // The stale file was replaced by a real snapshot, and the snapshot is a + // store a restore can put back: same rows, readable through the store. + if got := taskTitles(t, out); len(got) != 1 || got[0] != "before" { + t.Fatalf("snapshot tasks = %v, want the one seeded task", got) + } + if entries, err := filepath.Glob(out + ".tmp*"); err != nil || len(entries) != 0 { + t.Errorf("backup left scratch files beside the snapshot: %v (%v)", entries, err) + } +} + +// The snapshot is the rollback state, so an interrupted backup must never +// leave a truncated file where a good snapshot was. +func TestRecoveryBackupRefusesAMissingDatabase(t *testing.T) { + dir := t.TempDir() + code, _, stderr := runRecoveryCmd(t, "backup", "-db", filepath.Join(dir, "absent.sqlite"), "-out", backupPath(t, dir)) + if code == 0 { + t.Fatal("backup of a missing database succeeded") + } + if !strings.Contains(stderr, "absent.sqlite") { + t.Errorf("refusal must name the database it could not read; stderr = %q", stderr) + } +} + +// backup is the one recovery command allowed to run against a serving store, +// and that exemption is only sound while it never replaces the database. With +// -out naming the database it renames the snapshot over the live file and +// leaves the WAL of the database it just unlinked -- the state the restore path +// refuses up front and the code elsewhere calls unrecoverable. +func TestRecoveryBackupRefusesTheDatabaseAsItsOwnSnapshot(t *testing.T) { + db := filepath.Join(t.TempDir(), "archie.db-tasks.sqlite") + st := openTaskStore(t, db) + seedTask(t, st, "live") + + code, _, stderr := runRecoveryCmd(t, "backup", "-db", db, "-out", db) + if code == 0 { + t.Fatal("backup over the database itself succeeded") + } + if !strings.Contains(stderr, "is the database itself") { + t.Errorf("refusal must say the snapshot is the database; stderr = %q", stderr) + } + if got := taskTitles(t, db); len(got) != 1 || got[0] != "live" { + t.Fatalf("refused backup changed the database: tasks = %v", got) + } +} + +// Restoring is the forward-migration escape hatch: the previous release +// reads this file, so it must be exactly the snapshot, with the WAL that +// belongs to the replaced database gone rather than replayed over it. +func TestRecoveryRestorePutsTheSnapshotBackAndDropsItsWAL(t *testing.T) { + dir := t.TempDir() + db := filepath.Join(dir, "archie.db-tasks.sqlite") + out := backupPath(t, dir) + st := openTaskStore(t, db) + seedTask(t, st, "before") + backup(t, db, out) + seedTask(t, st, "after") + + code, stdout, stderr := runRecoveryCmd(t, "restore", "-db", db, "-from", out) + if code != 0 { + t.Fatalf("restore exited %d: %s", code, stderr) + } + if !strings.Contains(stdout, out) { + t.Errorf("restore must report the snapshot it restored from; stdout = %q", stdout) + } + // Checked before anything reopens the database: opening a store sets WAL + // mode, which creates these files, so the point is that the restore did not + // leave the replaced database's log behind -- not that a later reader + // cannot recreate its own. + for _, suffix := range []string{"-wal", "-shm"} { + if _, err := os.Stat(db + suffix); err == nil { + t.Errorf("%s survived the restore: the replaced database's write-ahead log must not be replayed over the snapshot", db+suffix) + } + } + if got := taskTitles(t, db); len(got) != 1 || got[0] != "before" { + t.Fatalf("restored tasks = %v, want only the snapshot's task", got) + } +} + +// Overwriting a database a running State Store still owns leaves that process +// writing to an unlinked inode while the store serves a file nobody owns -- +// the one way an offline command can destroy a task store. +func TestRecoveryRestoreRefusesAStoreTheStateStoreIsRunning(t *testing.T) { + dir := t.TempDir() + db := filepath.Join(dir, "archie.db-tasks.sqlite") + out := backupPath(t, dir) + st := openTaskStore(t, db) + seedTask(t, st, "before") + backup(t, db, out) + seedTask(t, st, "after") + holdStoreOwnership(t, db) + + code, _, stderr := runRecoveryCmd(t, "restore", "-db", db, "-from", out) + if code == 0 { + t.Fatal("restore against a live State Store succeeded") + } + if !strings.Contains(stderr, "owned by another process") { + t.Errorf("refusal must say the database is owned by another process; stderr = %q", stderr) + } + if got := taskTitles(t, db); len(got) != 2 { + t.Fatalf("refused restore changed the database: tasks = %v", got) + } +} + +func TestRecoveryRestoreRefusesAFileThatIsNotAStore(t *testing.T) { + dir := t.TempDir() + db := filepath.Join(dir, "archie.db-tasks.sqlite") + seedTask(t, openTaskStore(t, db), "live") + notAStore := filepath.Join(dir, "notes.txt") + if err := os.WriteFile(notAStore, []byte("this is not a database"), 0o600); err != nil { + t.Fatal(err) + } + + code, _, stderr := runRecoveryCmd(t, "restore", "-db", db, "-from", notAStore) + if code == 0 { + t.Fatal("restore from a file that is not a store succeeded") + } + if !strings.Contains(stderr, notAStore) { + t.Errorf("refusal must name the snapshot it could not read; stderr = %q", stderr) + } + if got := taskTitles(t, db); len(got) != 1 { + t.Fatalf("refused restore changed the database: tasks = %v", got) + } +} + +func TestRecoveryRestoreRefusesTheDatabaseAsItsOwnSnapshot(t *testing.T) { + // Restoring a database from itself would delete the write-ahead log of the + // file it is about to copy, so the one case that must never reach the swap + // is refused up front. + db := filepath.Join(t.TempDir(), "archie.db-tasks.sqlite") + seedTask(t, openTaskStore(t, db), "live") + + code, _, stderr := runRecoveryCmd(t, "restore", "-db", db, "-from", db) + if code == 0 { + t.Fatal("restore from the database itself succeeded") + } + if !strings.Contains(stderr, "is the database itself") { + t.Errorf("refusal must say the snapshot is the database; stderr = %q", stderr) + } + if got := taskTitles(t, db); len(got) != 1 { + t.Fatalf("refused restore changed the database: tasks = %v", got) + } +} + +func TestRecoveryValidateAcceptsAHealthyStore(t *testing.T) { + dir := t.TempDir() + configPath := writeMinimalConfig(t, dir) + db := filepath.Join(dir, "archie.db-tasks.sqlite") + st := openTaskStore(t, db) + seedTask(t, st, "healthy") + seedStoreResources(t, st, configPath) + + code, stdout, stderr := runRecoveryCmd(t, "validate", "-db", db, "-config", configPath) + if code != 0 { + t.Fatalf("validate exited %d: %s", code, stderr) + } + if !strings.Contains(stdout, "schema version") { + t.Errorf("validate must report the schema version it verified; stdout = %q", stdout) + } + // The count is part of what the operator reads, and a check that reports + // "0 stored resources" for a store full of them is worse than no check. + if count := storedResourceCount(t, stdout); count == 0 { + t.Errorf("validate reported no stored resources for a seeded store; stdout = %q", stdout) + } +} + +// validate's verdict is only useful if it is the daemon's verdict. Boot refuses +// on configuration.Validate over the file config with every stored resource +// layered onto it (internal/app/archied/control_plane.go), not on the write +// path's own per-resource decode, and the two disagree: the write path accepts +// a scheduling policy boot rejects. +func TestRecoveryValidateRunsTheGateBootRuns(t *testing.T) { + dir := t.TempDir() + configPath := writeMinimalConfig(t, dir) + db := filepath.Join(dir, "archie.db-tasks.sqlite") + seedStoreResources(t, openTaskStore(t, db), configPath) + + // The control: a store seeded from this config is one the daemon boots on. + code, _, stderr := runRecoveryCmd(t, "validate", "-db", db, "-config", configPath) + if code != 0 { + t.Fatalf("validate exited %d on a store the daemon starts on: %s", code, stderr) + } + + // A hand-edited value, or an older revision of one: the write path's own + // validator never inspects dispatch.trigger, so only the config gate + // catches it, and the daemon exits 1 on it (validateDispatch). + if err := execRaw(t, db, `UPDATE resources SET value='{"poll_interval":"1m","max_retries":3,"dispatch":{"trigger":"bogus"}}' WHERE kind='scheduling-policy'`); err != nil { + t.Fatal(err) + } + code, stdout, stderr := runRecoveryCmd(t, "validate", "-db", db, "-config", configPath) + if code == 0 { + t.Fatalf("validate accepted a stored policy the daemon refuses to boot with; stdout = %q", stdout) + } + if !strings.Contains(stderr, "dispatch.trigger") { + t.Errorf("refusal must name what boot rejects; stderr = %q", stderr) + } + + // A store the State Store never seeded holds no kinds at all. Its next start + // creates the resources table and seeds every kind from this same config + // (internal/app/archied/state_store.go), so archied layers those seeds and + // starts: refusing the file here would send an operator to restore a + // snapshot for a store the daemon boots on perfectly well. + unseeded := filepath.Join(dir, "unseeded.db-tasks.sqlite") + openTaskStore(t, unseeded) + code, stdout, stderr = runRecoveryCmd(t, "validate", "-db", unseeded, "-config", configPath) + if code != 0 { + t.Fatalf("validate refused a store the State Store would seed on its next start; stderr = %q", stderr) + } + if !strings.Contains(stdout, "0 stored resources validate") { + t.Errorf("validate must report that there was nothing stored to check; stdout = %q", stdout) + } + + // The upgrade case: a release defines a kind the store does not hold yet, + // and validate runs before the State Store has started again. The State + // Store seeds the kind it is missing, so this is the same stance as the + // store with nothing stored at all -- the file is not the reason the daemon + // would refuse to start, and saying otherwise costs the operator a snapshot. + partial := filepath.Join(dir, "partial.db-tasks.sqlite") + seedStoreResources(t, openTaskStore(t, partial), configPath) + if err := execRaw(t, partial, `DELETE FROM resources WHERE kind='provider-settings'`); err != nil { + t.Fatal(err) + } + code, stdout, stderr = runRecoveryCmd(t, "validate", "-db", partial, "-config", configPath) + if code != 0 { + t.Fatalf("validate refused a store missing a kind the State Store seeds; stderr = %q", stderr) + } + if count := storedResourceCount(t, stdout); count == 0 { + t.Errorf("validate must still check the kinds the store does hold; stdout = %q", stdout) + } +} + +// The subcommands advertise -h as the way to read their flags, and the serve +// path in the same binary exits 0 for the same request, so a script probing the +// recovery surface must not read a requested help as a failure. +func TestRecoveryHelpExitsZero(t *testing.T) { + commands := []string{archied.RecoveryBackup, archied.RecoveryRestore, archied.RecoveryValidate, archied.RecoveryRollback} + for _, command := range commands { + code, _, stderr := runRecoveryCmd(t, command, "-h") + if code != 0 { + t.Errorf("%s -h exited %d, want 0; stderr = %q", command, code, stderr) + } + } +} + +// validate diagnoses a deployment; it is not the deployment. Resolving the boot +// config must not create, append to, or rotate the daemon's log file or the +// directory it lives in: a line stamped component="daemon" in archied's log is +// indistinguishable from the daemon having written it. +func TestRecoveryValidateLeavesTheDaemonLogAlone(t *testing.T) { + dir := t.TempDir() + configPath, logFile := writeConfigWithLogFile(t, dir) + db := filepath.Join(dir, "archie.db-tasks.sqlite") + seedStoreResources(t, openTaskStore(t, db), configPath) + + code, _, stderr := runRecoveryCmd(t, "validate", "-db", db, "-config", configPath) + if code != 0 { + t.Fatalf("validate exited %d: %s", code, stderr) + } + if _, err := os.Stat(logFile); !os.IsNotExist(err) { + t.Fatalf("validate touched the daemon's log file %s (stat err = %v)", logFile, err) + } + if _, err := os.Stat(filepath.Dir(logFile)); !os.IsNotExist(err) { + t.Errorf("validate created the log directory %s (stat err = %v)", filepath.Dir(logFile), err) + } +} + +// validate's verdict is the daemon's verdict only if it reads the configuration +// the daemon reads. archied resolves that default from its own configHome, which +// honours a relative XDG_CONFIG_HOME; os.UserConfigDir rejects a relative one +// outright, which left the command with no config and no verdict at all. +func TestRecoveryDefaultConfigFollowsTheDaemonsConfigHome(t *testing.T) { + relative := filepath.Join("relative", "config") + t.Setenv("XDG_CONFIG_HOME", relative) + if got, want := defaultConfigPath(), filepath.Join(relative, "archie", "config.toml"); got != want { + t.Errorf("defaultConfigPath() with a relative XDG_CONFIG_HOME = %q, want %q", got, want) + } + + absolute := t.TempDir() + t.Setenv("XDG_CONFIG_HOME", absolute) + if got, want := defaultConfigPath(), filepath.Join(absolute, "archie", "config.toml"); got != want { + t.Errorf("defaultConfigPath() = %q, want %q", got, want) + } + // The equality itself: one helper, so the two cannot drift apart again. + if got, want := defaultConfigPath(), archied.DefaultConfigPath(); got != want { + t.Errorf("defaultConfigPath() = %q, but the daemon resolves %q", got, want) + } +} + +// validate exists to answer "would archied start against this file". Each case +// below is a state the daemon fails closed on, or a file it cannot read at all. +func TestRecoveryValidateRefusesWhatTheDaemonRefuses(t *testing.T) { + newStore := func(t *testing.T) string { + t.Helper() + db := filepath.Join(t.TempDir(), "archie.db-tasks.sqlite") + openTaskStore(t, db) + return db + } + + t.Run("forward_migrated_schema", func(t *testing.T) { + // A database written by a newer release is the state a rollback has to + // recognise before it hands the file to an older binary. + db := newStore(t) + if err := execRaw(t, db, `PRAGMA user_version = 99`); err != nil { + t.Fatal(err) + } + code, _, stderr := runRecoveryCmd(t, "validate", "-db", db) + if code == 0 { + t.Fatal("validate accepted a forward-migrated database") + } + if !strings.Contains(stderr, "newer than supported version") { + t.Errorf("refusal must name the schema version; stderr = %q", stderr) + } + }) + + t.Run("corrupt_file", func(t *testing.T) { + // A real database header over pages that are not a database. The magic + // is right, so this fails on the content rather than on the format, which + // is the shape a half-written or truncated store has. + db := filepath.Join(t.TempDir(), "archie.db-tasks.sqlite") + corrupt := append([]byte("SQLite format 3\x00"), bytes.Repeat([]byte{0xff}, 4096)...) + if err := os.WriteFile(db, corrupt, 0o600); err != nil { + t.Fatal(err) + } + code, _, stderr := runRecoveryCmd(t, "validate", "-db", db) + if code == 0 { + t.Fatal("validate accepted a corrupt database") + } + if !strings.Contains(stderr, db) { + t.Errorf("refusal must name the file; stderr = %q", stderr) + } + }) + + t.Run("not_a_database", func(t *testing.T) { + // Pointing -db at the wrong file is the common operator mistake, and the + // configured db_path is a different file from the task database the + // State Store owns. + db := filepath.Join(t.TempDir(), "archie.db") + if err := os.WriteFile(db, []byte("db_path, not the task database"), 0o600); err != nil { + t.Fatal(err) + } + code, _, stderr := runRecoveryCmd(t, "validate", "-db", db) + if code == 0 { + t.Fatal("validate accepted a file that is not a database") + } + if !strings.Contains(stderr, db) { + t.Errorf("refusal must name the file; stderr = %q", stderr) + } + }) + + t.Run("stored_resource_that_will_not_validate", func(t *testing.T) { + // A stored value that both gates refuse: the write path stops it at + // Decode (workflow.ExecutionSettings.Validate rejects a negative limit), + // and boot stops on it too -- startWorkflowExecutionSettings decodes the + // same document and exits 1. It can only get here by bypassing the write + // path, which is exactly the state the operator has no path back from. + db := newStore(t) + seedResource(t, openTaskStore(t, db), controlplane.WorkflowExecutionSettingsKind, + `{"max_model_tool_steps":-1,"max_runtime_seconds":60,"max_consecutive_gate_failures":3}`, "hand-edited") + code, _, stderr := runRecoveryCmd(t, "validate", "-db", db) + if code == 0 { + t.Fatal("validate accepted a stored resource the write path would refuse") + } + if !strings.Contains(stderr, controlplane.WorkflowExecutionSettingsKind) { + t.Errorf("refusal must name the resource kind; stderr = %q", stderr) + } + }) + + t.Run("missing_database", func(t *testing.T) { + db := newStore(t) + if err := os.Remove(db); err != nil { + t.Fatal(err) + } + code, _, stderr := runRecoveryCmd(t, "validate", "-db", db) + if code == 0 { + t.Fatal("validate accepted a database that does not exist") + } + if !strings.Contains(stderr, db) { + t.Errorf("refusal must name the file; stderr = %q", stderr) + } + }) + + t.Run("store_without_the_control_plane_table", func(t *testing.T) { + // A store written before the control plane existed holds no stored + // settings. The serving process creates the table on its next start, so + // refusing this file would send an operator to restore a snapshot for a + // database that would have started perfectly well. + dir := t.TempDir() + configPath := writeMinimalConfig(t, dir) + db := filepath.Join(dir, "archie.db-tasks.sqlite") + openTaskStore(t, db) + if err := execRaw(t, db, `DROP TABLE resources`); err != nil { + t.Fatal(err) + } + code, stdout, stderr := runRecoveryCmd(t, "validate", "-db", db, "-config", configPath) + if code != 0 { + t.Fatalf("validate exited %d: %s", code, stderr) + } + if !strings.Contains(stdout, "0 stored resources validate") { + t.Errorf("validate must report that there was nothing stored to check; stdout = %q", stdout) + } + }) +} + +// rollback is the one operation that closes the documented gap: with the State +// Store stopped, it replays the value an earlier revision recorded through the +// ordinary replace, so no rollback RPC is needed and the rollback itself is +// audited as one more revision. +func TestRecoveryRollbackReplaysTheRevisionThroughReplace(t *testing.T) { + dir := t.TempDir() + db := filepath.Join(dir, "archie.db-tasks.sqlite") + st := openTaskStore(t, db) + seedResource(t, st, controlplane.ModelRoleAssignmentsKind, `{"implement":"anthropic/claude"}`, "first") + seedResource(t, st, controlplane.ModelRoleAssignmentsKind, `{"implement":"openai/gpt"}`, "second") + + code, stdout, stderr := runRecoveryCmd(t, "rollback", "-db", db, "-kind", controlplane.ModelRoleAssignmentsKind) + if code != 0 { + t.Fatalf("rollback exited %d: %s", code, stderr) + } + if !strings.Contains(stdout, controlplane.ModelRoleAssignmentsKind) { + t.Errorf("rollback must name the resource it replayed; stdout = %q", stdout) + } + resource, err := st.Resource(t.Context(), controlplane.ModelRoleAssignmentsKind) + if err != nil { + t.Fatal(err) + } + if string(resource.Value) != `{"implement":"anthropic/claude"}` { + t.Errorf("resource value = %s, want the first revision's value", resource.Value) + } + if resource.Version != 3 { + t.Errorf("resource version = %d, want a new revision (3) rather than a rewrite of the old one", resource.Version) + } + history, err := st.ResourceHistory(t.Context(), controlplane.ModelRoleAssignmentsKind, 0) + if err != nil { + t.Fatal(err) + } + if len(history) != 3 { + t.Fatalf("history has %d revisions, want the rollback recorded as the third", len(history)) + } + if history[0].Source != archied.OfflineRollbackSource { + t.Errorf("newest revision source = %q, want %q so the rollback is auditable", history[0].Source, archied.OfflineRollbackSource) + } +} + +// The revision to restore may be named, which is how an operator reaches past +// the most recent change without replaying it first. +func TestRecoveryRollbackRestoresANamedRevision(t *testing.T) { + dir := t.TempDir() + db := filepath.Join(dir, "archie.db-tasks.sqlite") + st := openTaskStore(t, db) + seedResource(t, st, controlplane.ModelRoleAssignmentsKind, `{"implement":"anthropic/claude"}`, "first") + seedResource(t, st, controlplane.ModelRoleAssignmentsKind, `{"implement":"openai/gpt"}`, "second") + seedResource(t, st, controlplane.ModelRoleAssignmentsKind, `{"implement":"google/gemini"}`, "third") + + code, _, stderr := runRecoveryCmd(t, "rollback", "-db", db, "-kind", controlplane.ModelRoleAssignmentsKind, "-revision", "1") + if code != 0 { + t.Fatalf("rollback exited %d: %s", code, stderr) + } + resource, err := st.Resource(t.Context(), controlplane.ModelRoleAssignmentsKind) + if err != nil { + t.Fatal(err) + } + if string(resource.Value) != `{"implement":"anthropic/claude"}` { + t.Errorf("resource value = %s, want the value recorded at revision 1", resource.Value) + } +} + +func TestRecoveryRollbackRefusesWhatItCannotReplay(t *testing.T) { + newStore := func(t *testing.T, revisions int) string { + t.Helper() + db := filepath.Join(t.TempDir(), "archie.db-tasks.sqlite") + st := openTaskStore(t, db) + for i := range revisions { + value := `{"implement":"anthropic/claude"}` + if i > 0 { + value = `{"implement":"openai/gpt"}` + } + seedResource(t, st, controlplane.ModelRoleAssignmentsKind, value, "seed") + } + return db + } + + t.Run("unknown_kind", func(t *testing.T) { + db := newStore(t, 2) + code, _, stderr := runRecoveryCmd(t, "rollback", "-db", db, "-kind", "not-a-resource") + if code == 0 { + t.Fatal("rollback of an unknown resource kind succeeded") + } + if !strings.Contains(stderr, "not-a-resource") { + t.Errorf("refusal must name the kind; stderr = %q", stderr) + } + }) + + t.Run("no_earlier_revision", func(t *testing.T) { + db := newStore(t, 1) + code, _, stderr := runRecoveryCmd(t, "rollback", "-db", db, "-kind", controlplane.ModelRoleAssignmentsKind) + if code == 0 { + t.Fatal("rollback with no earlier revision succeeded") + } + if !strings.Contains(stderr, "no earlier revision") { + t.Errorf("refusal must say there is nothing to roll back to; stderr = %q", stderr) + } + }) + + t.Run("current_revision", func(t *testing.T) { + db := newStore(t, 2) + code, _, stderr := runRecoveryCmd(t, "rollback", "-db", db, "-kind", controlplane.ModelRoleAssignmentsKind, "-revision", "2") + if code == 0 { + t.Fatal("rollback of the current revision succeeded") + } + if !strings.Contains(stderr, "is the current version") { + t.Errorf("refusal must say the revision is already the current value; stderr = %q", stderr) + } + }) + + t.Run("live_state_store", func(t *testing.T) { + db := newStore(t, 2) + holdStoreOwnership(t, db) + code, _, stderr := runRecoveryCmd(t, "rollback", "-db", db, "-kind", controlplane.ModelRoleAssignmentsKind) + if code == 0 { + t.Fatal("rollback against a live State Store succeeded") + } + if !strings.Contains(stderr, "owned by another process") { + t.Errorf("refusal must say the database is owned by another process; stderr = %q", stderr) + } + }) +} + +// A positional argument that is not a subcommand used to be ignored, leaving +// an operator who mistyped a command watching a server start instead. +func TestRecoveryRejectsAnUnknownSubcommand(t *testing.T) { + var stdout, stderr bytes.Buffer + code := runRecovery([]string{"backp", "-db", "x"}, &stdout, &stderr) + if code == 0 { + t.Fatal("an unknown subcommand exited 0") + } + if !strings.Contains(stderr.String(), "backp") { + t.Errorf("refusal must name the unknown subcommand; stderr = %q", stderr.String()) + } +} + +// writeConfigWithLogFile is writeMinimalConfig plus the daemon's durable log +// destination, which only the daemon may bring into existence. +func writeConfigWithLogFile(t *testing.T, dir string) (string, string) { + t.Helper() + configPath := writeMinimalConfig(t, dir) + logFile := filepath.Join(dir, "logs", "archied.log") + f, err := os.OpenFile(configPath, os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + t.Fatalf("open config to append: %v", err) + } + defer func() { + if err := f.Close(); err != nil { + t.Fatalf("close config: %v", err) + } + }() + if _, err := fmt.Fprintf(f, "\n[log]\nfile = %q\n", logFile); err != nil { + t.Fatalf("append [log]: %v", err) + } + return configPath, logFile +} + +// seedStoreResources seeds the store the way the State Store seeds it at boot, +// from the same config file the operator hands validate: the command's verdict +// is about the pair, and a store whose settings never matched the config is a +// state boot does not recognise either. +// +// The seeding builds its control-plane server from the shared provider set +// (internal/infrastructure/workflowsteps), which is what every composition root +// registers -- including openStateStoreControlPlane, the root helper the +// recovery path uses. This test cannot call that unexported helper, so it +// registers the same set itself rather than an empty manager: the definitions +// are decoded with the vocabulary the served path validates against, which is +// what makes the seeded values the ones validate is asked about. +func seedStoreResources(t *testing.T, st *store.Store, configPath string) { + t.Helper() + doc, err := configuration.New(nil).Resolve(configPath, "") + if err != nil { + t.Fatalf("resolve %s: %v", configPath, err) + } + steps, err := workflowsteps.NewManager() + if err != nil { + t.Fatalf("register workflow step vocabulary: %v", err) + } + server, err := controlplane.NewServer(st, steps) + if err != nil { + t.Fatalf("build control plane server: %v", err) + } + if _, err := server.ImportConfig(t.Context(), doc.Config); err != nil { + t.Fatalf("seed store resources: %v", err) + } +} + +// storedResourceCount reads the count validate reports out of its summary line. +func storedResourceCount(t *testing.T, stdout string) int { + t.Helper() + before, found := strings.CutSuffix(stdout, " stored resources validate\n") + if !found { + t.Fatalf("no resource count in %q", stdout) + } + fields := strings.Fields(before) + count, err := strconv.Atoi(fields[len(fields)-1]) + if err != nil { + t.Fatalf("resource count in %q: %v", stdout, err) + } + return count +} + +// seedResource writes one revision of a control-plane resource through the +// store, the way the control plane's replace does. The request ID is derived +// from the label and the revision it produces: the store deduplicates a +// repeated request ID, so a fixed one would make the second write of a series +// a silent no-op. +func seedResource(t *testing.T, st *store.Store, kind, value, label string) { + t.Helper() + current, err := st.Resource(t.Context(), kind) + expected := int64(0) + if err == nil { + expected = current.Version + } + var document any + if err := json.Unmarshal([]byte(value), &document); err != nil { + t.Fatal(err) + } + encoded, err := json.Marshal(document) + if err != nil { + t.Fatal(err) + } + if _, err := st.PutResource(t.Context(), store.ResourceWrite{ + Kind: kind, Value: encoded, Actor: label, Source: "test", + RequestID: fmt.Sprintf("%s-%d", label, expected+1), ExpectedVersion: expected, + }); err != nil { + t.Fatalf("seed resource %s: %v", kind, err) + } +} + +// execRaw runs one statement against the database file the way an operator +// editing a broken store would, bypassing every check the store applies. +func execRaw(t *testing.T, path, statement string) error { + t.Helper() + db, err := sql.Open("sqlite", path) + if err != nil { + return err + } + defer func() { _ = db.Close() }() + _, err = db.ExecContext(t.Context(), statement) + return err +} diff --git a/docs/architecture/safe-change-and-recovery.md b/docs/architecture/safe-change-and-recovery.md index 564e27e00..a1bb55e4e 100644 --- a/docs/architecture/safe-change-and-recovery.md +++ b/docs/architecture/safe-change-and-recovery.md @@ -120,21 +120,50 @@ file values would restore the second source of truth the control plane exists to remove. The cost is that a bad stored value stops the daemon starting, and the value can only be corrected through the daemon's own API. -### Recovery: not yet built +### Recovery: offline commands on the State Store -There is no escape hatch. Nothing skips the control plane at boot, and the only -writer for a resource is `ControlPlaneService.Command` on a running State Store. -An operator whose stored settings will not validate has no supported path back -short of editing `archie.db` by hand. +Nothing skips the control plane at boot, and the only writer for a resource is +`ControlPlaneService.Command` on a running State Store -- which is exactly what +the daemon fails closed without. `archie-state-store` therefore carries the +offline recovery commands (`archie-core-6zw0`). They work on the task database +file directly -- the configured `db_path` with `-tasks.sqlite` appended, not +`db_path` itself -- and need neither `archied` nor the Web UI running: `validate` +reads the configuration file the daemon boots with, which is what makes its +verdict the daemon's verdict. -`archie-core-6zw0` is the missing piece: offline commands that back up, restore, -validate and roll back the database without `archied` or the Web UI. Until it -lands, treat a settings change that requires a restart as a change that can -prevent one, and take a copy of `archie.db` first. +| Command | What it does | State Store | +| --- | --- | --- | +| `backup -db F -out F` | writes a transactionally consistent snapshot (`VACUUM INTO`), verified as a usable store before it is published | may be running | +| `restore -db F -from F` | replaces the database with a snapshot and drops the replaced file's `-wal`/`-shm` | must be stopped | +| `validate -db F [-config C]` | runs the checks that stop archied starting: the file is readable, uncorrupted and no newer than the binary; every stored resource decodes under the definition that owns it; and `configuration.Validate` accepts the settings the daemon would read -- the stored ones, or the seed the State Store writes for a kind the file does not hold yet -- layered onto `C`, the check the daemon reports as `validate database settings` | may be running | +| `rollback -db F -kind K [-revision N]` | replays an earlier revision of a stored resource through the ordinary replace, recording the rollback as a new revision | must be stopped | -Resource history is the in-band remedy and does exist: every write records its -prior value, so an edit made through the dashboard or a channel can be restored -by replaying an earlier revision while the State Store is up. +The two validation layers `validate` reports are not the same check, and it +reports both rather than merging them: the write path's own decode is stricter +about shape (it rejects unknown fields boot's decode ignores), while boot is +stricter about meaning (only boot checks `dispatch.trigger`, and a positive +`poll_interval`). Either one refusing is a store somebody has to look at. + +No rollback RPC exists and none is needed: a revision carries its own value, so +restoring one is the same replace the dashboard and the channels already use. +The offline command is that operation with the store stopped -- the one state in +which the daemon's own API cannot serve it. + +"Must be stopped" is checked rather than assumed. The serving process takes an +exclusive flock on `.lock` for its whole life, and `restore` and `rollback` +refuse with `task database is owned by another process` while another process +holds it. The read-only commands -- `backup` and `validate` -- take no lock and +are meant to run against a serving store: `backup` because the update installer +runs inside the process it is replacing and cannot stop the State Store, and +`validate` because an operator checks a running deployment with it. That +`backup` exemption is only sound while backup never replaces the database, so +naming the database as its own `-out` is refused. `scripts/archie-update-install` +takes its pre-flight snapshot through this subcommand rather than through its +own `sqlite3` shell, so the installer and an operator run one implementation. + +`rollback` is the surgical route back from a bad stored value; `restore` is the +blunt one, and is also the way back from a schema migration a release cannot +grow out of. ## Gateway restart: two constraints learned the hard way diff --git a/internal/app/archied/bootstrap.go b/internal/app/archied/bootstrap.go index 81d22d283..bc67a1147 100644 --- a/internal/app/archied/bootstrap.go +++ b/internal/app/archied/bootstrap.go @@ -83,6 +83,13 @@ type boot struct { cfg config.Config log *slog.Logger + // stderrLog keeps setupLogging on stderr, never touching cfg.Log.File. An + // offline command reads the bootstrap config but is not the daemon: creating + // the deployment's log, appending a line stamped component="daemon", or + // rotating a log the daemon is writing are side effects a diagnosis must not + // have, and it needs no durable copy of its own. + stderrLog bool + loader *configuration.Loader doc *configuration.Document currentProvenance atomic.Pointer[configuration.Provenance] @@ -297,6 +304,12 @@ func (b *boot) loadConfig(ctx context.Context, cfgPath, overlayPath string) erro // must not take the daemon down with it. func (b *boot) setupLogging() error { cfg := b.cfg + if b.stderrLog { + // The stderr logger newBootstrap installed is the whole destination. + // Nothing below this line is set up: the feed, the task-log registry and + // the rotating file all belong to a running daemon. + return nil + } logFeed := logging.NewFeed(1000) b.logFeed = logFeed // Task logs live alongside the store rather than under cfg.Log.File's diff --git a/internal/app/archied/main.go b/internal/app/archied/main.go index 37870ec3f..3ea12fbe2 100644 --- a/internal/app/archied/main.go +++ b/internal/app/archied/main.go @@ -315,7 +315,7 @@ type runArgs struct { } func parseArgs() (runArgs, bool) { - defaultCfg := filepath.Join(configHome(), "archie", "config.toml") + defaultCfg := DefaultConfigPath() var args runArgs flag.StringVar(&args.cfgPath, "config", defaultCfg, "path to a TOML/YAML config file or configuration directory") flag.StringVar(&args.overlayPath, "config-overlay", "", "path to a TOML/YAML overlay file or configuration directory applied on top of -config") @@ -918,6 +918,15 @@ func configHome() string { return filepath.Join(home, ".config") } +// DefaultConfigPath is the configuration the daemon reads when -config is not +// given, and the one every command that asks a question about the daemon has to +// read to be answering about the same deployment: it is the daemon's rule, not +// a second one. It is exported for exactly that reason -- an offline command +// resolving the default any other way answers about a different config file +// (os.UserConfigDir, notably, rejects a relative XDG_CONFIG_HOME outright and +// leaves the caller with no path at all). +func DefaultConfigPath() string { return filepath.Join(configHome(), "archie", "config.toml") } + // updateReportPath is where the update watchdog leaves the phase-2 outcome // of an update for this identity to relay on its next launch. The identity is // hashed so multiple identities sharing one daemon (see diff --git a/internal/app/archied/state_store.go b/internal/app/archied/state_store.go index d2a1da380..b1b919d35 100644 --- a/internal/app/archied/state_store.go +++ b/internal/app/archied/state_store.go @@ -215,7 +215,25 @@ func (b *boot) openStateStore(ctx context.Context) error { log.Error("configure bindings cipher", "err", err) return err } - st, err := openProductionTaskStore(ctx, taskDBPath(cfg.DBPath), store.WithBindingCipher(bindingCipher)) + // Claim the database before opening it. This process is the file's owner + // for as long as it lives, and the claim is what an offline recovery + // command checks before it rewrites the file: without it, "the State Store + // is stopped" is a rule the operator is trusted to have followed rather + // than a fact the command can verify. A second State Store on the same + // database fails here, which is the single-owner invariant stated instead + // of merely documented. + path := taskDBPath(cfg.DBPath) + ownership, err := store.AcquireOwnership(path) + if err != nil { + log.Error("claim state store ownership", "err", err) + return err + } + b.addCleanup(func() { + if err := ownership.Release(); err != nil { + log.Error("release state store ownership", "err", err) + } + }) + st, err := openProductionTaskStore(ctx, path, store.WithBindingCipher(bindingCipher)) if err != nil { log.Error("open state store", "err", err) return err diff --git a/internal/app/archied/state_store_recovery.go b/internal/app/archied/state_store_recovery.go new file mode 100644 index 000000000..fe4636726 --- /dev/null +++ b/internal/app/archied/state_store_recovery.go @@ -0,0 +1,280 @@ +// state_store_recovery.go composes the offline recovery subcommands of the +// standalone archie-state-store process. They exist because the control plane +// fails closed: a stored setting that will not validate stops archied +// starting, and the in-band remedy -- replaying an earlier revision while the +// State Store is up -- is unavailable in exactly the case that needs it +// (docs/architecture/safe-change-and-recovery.md). +// +// Every operation works on the task database file directly, with the State +// Store stopped, and the ones that write refuse when another process owns the +// file. Backup is the exception and deliberately so: the update installer +// cannot stop the process that runs it, so the snapshot that makes a failed +// update reversible is taken against a serving store through SQLite's own +// consistent copy. +// +// The control-plane server both validate and rollback read through is built by +// openStateStoreControlPlane -- the same root helper the served path uses -- so +// the offline checks resolve the same workflow step vocabulary as the process +// whose verdict they predict, and a constructor error is a refusal here rather +// than a server that would validate against a different vocabulary. +package archied + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/samcharles93/archie-core/internal/config" + pb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1" + "github.com/samcharles93/archie-core/internal/infrastructure/configuration" + "github.com/samcharles93/archie-core/internal/store" +) + +// The audit identity every offline rollback records. A rollback is one more +// revision of the resource, written through the ordinary replace, so the +// operator can see who went back and when -- and can go forward again the same +// way, with the State Store up or down. +const ( + OfflineRollbackSource = "archie-state-store rollback" + OfflineRollbackActor = "operator:offline-recovery" +) + +// Recovery operations, as named on the command line. +const ( + RecoveryBackup = "backup" + RecoveryRestore = "restore" + RecoveryValidate = "validate" + RecoveryRollback = "rollback" +) + +// StateStoreRecoveryOptions are the process inputs for one offline recovery +// operation. +type StateStoreRecoveryOptions struct { + // Operation is one of backup, restore, validate, rollback. + Operation string + // DB is the task database file the State Store owns: the configured + // db_path with "-tasks.sqlite" appended, not the configured path itself. + DB string + // Out is the snapshot backup writes. + Out string + // From is the snapshot restore reads. + From string + // Config and Overlay are the configuration the daemon would boot with. + // validate asks whether the stored settings are a state archied starts on, + // and that question is about the stored values *layered onto this + // configuration*, so the file config is part of the answer. + Config string + Overlay string + // Kind is the control-plane resource kind rollback replays. + Kind string + // Revision is the revision rollback replays. Zero means the newest + // revision older than the one the resource carries now. + Revision int64 +} + +// RunStateStoreRecovery performs one offline operation on the task database and +// returns the line the command reports on stdout. +func RunStateStoreRecovery(ctx context.Context, options StateStoreRecoveryOptions) (string, error) { + switch options.Operation { + case RecoveryBackup: + if options.DB == "" || options.Out == "" { + return "", errors.New("backup requires -db and -out") + } + if err := store.Backup(ctx, options.DB, options.Out); err != nil { + return "", err + } + return fmt.Sprintf("backed up %s to %s", options.DB, options.Out), nil + case RecoveryRestore: + if options.DB == "" || options.From == "" { + return "", errors.New("restore requires -db and -from") + } + if err := store.Restore(ctx, options.DB, options.From); err != nil { + return "", err + } + return fmt.Sprintf("restored %s from %s; start the State Store again", options.DB, options.From), nil + case RecoveryValidate: + if options.DB == "" { + return "", errors.New("validate requires -db") + } + return validateStore(ctx, options) + case RecoveryRollback: + if options.DB == "" || options.Kind == "" { + return "", errors.New("rollback requires -db and -kind") + } + return rollbackResource(ctx, options) + default: + return "", fmt.Errorf("unknown recovery command %q", options.Operation) + } +} + +// validateStore answers the question an operator actually has: would archied +// start against this file. It runs the checks that stop it, in the order boot +// meets them. +// +// First the file itself, since a database that is corrupt, newer than this +// binary, or not a database is one the serving process refuses to open. Then +// the writer's own per-resource validation, which is what a hand-edited value +// violated. Then boot's gate: the file config with every stored resource +// layered onto it, and configuration.Validate -- the check whose failure the +// daemon reports as "validate database settings". +// +// The two validation layers disagree in both directions, and both are reported +// rather than merged. The write path is stricter about shape (it rejects +// unknown fields boot's decode ignores), while boot is stricter about meaning +// (only boot checks dispatch.trigger and a positive poll interval, both of +// which the write path's own validators let through). A store either check +// refuses is a store somebody has to look at. +func validateStore(ctx context.Context, options StateStoreRecoveryOptions) (string, error) { + version, err := store.ValidateFile(ctx, options.DB) + if err != nil { + return "", err + } + st, err := store.OpenReadOnly(ctx, options.DB) + if err != nil { + return "", err + } + defer func() { _ = st.Close() }() + + server, err := openStateStoreControlPlane(st) + if err != nil { + return "", err + } + // Is there a resources table at all? A store written before the control plane + // existed has none, and the State Store creates it and seeds every kind from + // this same config on its next start -- so there is nothing stored to check + // with the writer's validator, and boot's gate is the config check alone. + // That is the same stance the layering takes for a kind the table is missing: + // what the daemon reads for a kind nothing was ever stored for is the seed. + checked := 0 + stored, err := st.StoresResources(ctx) + if err != nil { + return "", err + } + if stored { + if checked, err = server.ValidateStored(ctx); err != nil { + return "", err + } + } + + base, err := bootConfig(ctx, options) + if err != nil { + return "", err + } + document := base + if stored { + document, _, err = server.StoredRuntimeConfig(ctx, base) + if err != nil { + return "", err + } + } + if err := configuration.Validate(&document); err != nil { + // The daemon's own wording, so an operator can match this failure to the + // one that stopped archied starting. + return "", fmt.Errorf("validate database settings: %w", err) + } + return fmt.Sprintf("%s is a valid store at schema version %d; %d stored resources validate", options.DB, version, checked), nil +} + +// bootConfig resolves the configuration the daemon would boot with. validate is +// the only recovery command that asks a question about the process rather than +// the file, and that question is about the stored values layered onto this +// document, so the document is part of the answer. +// +// It reads that config with the daemon's stderr logger: resolving it is a +// diagnosis, and a diagnosis that opened, appended to or rotated cfg.Log.File +// would edit the deployment it is inspecting. +func bootConfig(ctx context.Context, options StateStoreRecoveryOptions) (config.Config, error) { + b := newBootstrap() + b.stderrLog = true + defer b.cleanup() + if err := b.loadConfig(ctx, options.Config, options.Overlay); err != nil { + return config.Config{}, err + } + return b.cfg, nil +} + +// rollbackResource is the one operation the Web UI cannot be asked to perform +// here: the daemon fails closed on the bad value, so the store that would serve +// the rollback is the store that will not start. It replays the revision's own +// value through the ordinary replace, so no rollback RPC exists or is needed, +// and the rollback is audited as one more revision. +func rollbackResource(ctx context.Context, options StateStoreRecoveryOptions) (string, error) { + // Before the lock and before the open: opening a store creates the file, + // and a rollback pointed at a mistyped path must not leave an empty + // database where the operator expected theirs. + if err := store.RequireDatabase(options.DB); err != nil { + return "", err + } + ownership, err := store.AcquireOwnership(options.DB) + if err != nil { + return "", err + } + defer func() { _ = ownership.Release() }() + + st, err := store.Open(ctx, options.DB) + if err != nil { + return "", err + } + defer func() { _ = st.Close() }() + + server, err := openStateStoreControlPlane(st) + if err != nil { + return "", err + } + if !server.Owns(options.Kind) { + return "", fmt.Errorf("unknown resource kind %q", options.Kind) + } + current, err := st.Resource(ctx, options.Kind) + if errors.Is(err, store.ErrResourceNotFound) { + return "", fmt.Errorf("%s has no stored value to roll back", options.Kind) + } + if err != nil { + return "", fmt.Errorf("read %s: %w", options.Kind, err) + } + revision, err := revisionToReplay(ctx, st, options.Kind, current.Version, options.Revision) + if err != nil { + return "", err + } + replaced, err := server.Command(ctx, &pb.CommandRequest{ + Kind: options.Kind, + Command: "replace", + ValueJson: revision.Value, + ExpectedVersion: current.Version, + Actor: OfflineRollbackActor, + Source: OfflineRollbackSource, + RequestId: fmt.Sprintf("recovery-rollback-%s-%d", options.Kind, time.Now().UTC().UnixNano()), + }) + if err != nil { + return "", fmt.Errorf("replay %s at revision %d: %w", options.Kind, revision.Version, err) + } + return fmt.Sprintf("rolled back %s from version %d to the value recorded at version %d; the store now holds version %d", + options.Kind, current.Version, revision.Version, replaced.GetResource().GetVersion()), nil +} + +// revisionToReplay picks the value to put back: the revision the operator +// named, or the newest one older than the value the resource carries now. +func revisionToReplay(ctx context.Context, st *store.Store, kind string, current, requested int64) (store.Resource, error) { + history, err := st.ResourceHistory(ctx, kind, 0) + if err != nil { + return store.Resource{}, fmt.Errorf("read %s history: %w", kind, err) + } + if requested > 0 { + for _, revision := range history { + if revision.Version != requested { + continue + } + if revision.Version == current { + return store.Resource{}, fmt.Errorf("%s revision %d is the current version; there is nothing to roll back to", kind, requested) + } + return revision, nil + } + return store.Resource{}, fmt.Errorf("%s has no revision %d", kind, requested) + } + for _, revision := range history { + if revision.Version < current { + return revision, nil + } + } + return store.Resource{}, fmt.Errorf("%s has no earlier revision to roll back to", kind) +} diff --git a/internal/app/controlplane/registry.go b/internal/app/controlplane/registry.go index b49186ce9..629421c93 100644 --- a/internal/app/controlplane/registry.go +++ b/internal/app/controlplane/registry.go @@ -72,13 +72,9 @@ func (s *Server) ImportConfig(ctx context.Context, cfg config.Config) (map[strin if !errors.Is(err, store.ErrResourceNotFound) { return nil, err } - value, err := json.Marshal(definition.Seed(cfg)) + value, err := definition.seededValue(cfg) if err != nil { - return nil, fmt.Errorf("seed %s: %w", definition.Kind, err) - } - value, err = definition.Decode(value) - if err != nil { - return nil, fmt.Errorf("seed %s: %w", definition.Kind, err) + return nil, err } resource, err = s.store.PutResource(ctx, store.ResourceWrite{Kind: definition.Kind, Value: value, Actor: "system:migration", Source: "legacy-config", RequestID: "import:" + definition.Kind, ExpectedVersion: 0, At: time.Now().UTC()}) if err != nil { @@ -89,6 +85,76 @@ func (s *Server) ImportConfig(ctx context.Context, cfg config.Config) (map[strin return versions, nil } +// seededValue is the value the State Store writes for a kind it does not hold +// yet: the definition's seed derived from cfg, run through the same Decode a +// write applies. ImportConfig stores it; the offline config layering reads it +// for the same reason, so "what the daemon sees for a kind that was never +// stored" is one value rather than two. +func (d Definition) seededValue(cfg config.Config) ([]byte, error) { + value, err := json.Marshal(d.Seed(cfg)) + if err != nil { + return nil, fmt.Errorf("seed %s: %w", d.Kind, err) + } + value, err = d.Decode(value) + if err != nil { + return nil, fmt.Errorf("seed %s: %w", d.Kind, err) + } + return value, nil +} + +// seededValues is what the State Store would write for every kind, keyed by +// kind. +func (s *Server) seededValues(cfg config.Config) (map[string][]byte, error) { + values := make(map[string][]byte, len(s.ordered)) + for _, definition := range s.ordered { + value, err := definition.seededValue(cfg) + if err != nil { + return nil, err + } + values[definition.Kind] = value + } + return values, nil +} + +// Owns reports whether kind is a resource the control plane owns, so a caller +// acting on a stored value can refuse a kind nobody defines before it reaches +// the store. +func (s *Server) Owns(kind string) bool { + _, ok := s.definitions[kind] + return ok +} + +// ValidateStored decodes every stored resource with the definition that owns +// it, using the same Decode the write path applies, and reports how many +// resources it checked. Errors name the kind and the revision it was stored at, +// which is what an operator needs to roll the value back. +// +// It is the write path's own check, not boot's: boot refuses on +// configuration.Validate over the stored values layered onto the file config, +// and the two disagree in both directions (this one rejects unknown fields +// boot's decode ignores, while only boot checks dispatch.trigger and a positive +// poll interval). A caller answering "would the daemon start" therefore needs +// StoredRuntimeConfig and configuration.Validate as well -- see +// validateStore in internal/app/archied/state_store_recovery.go. +func (s *Server) ValidateStored(ctx context.Context) (int, error) { + checked := 0 + var failures []error + for _, definition := range s.ordered { + resource, err := s.store.Resource(ctx, definition.Kind) + if errors.Is(err, store.ErrResourceNotFound) { + continue + } + if err != nil { + return checked, fmt.Errorf("read %s: %w", definition.Kind, err) + } + checked++ + if _, err := definition.Decode(resource.Value); err != nil { + failures = append(failures, fmt.Errorf("%s at version %d: %w", definition.Kind, resource.Version, err)) + } + } + return checked, errors.Join(failures...) +} + func validateAs[T any](input []byte, validate func(T) error) error { var value T decoder := json.NewDecoder(bytes.NewReader(input)) diff --git a/internal/app/controlplane/runtime_config.go b/internal/app/controlplane/runtime_config.go index c98f0d86e..0bd90c6c5 100644 --- a/internal/app/controlplane/runtime_config.go +++ b/internal/app/controlplane/runtime_config.go @@ -3,13 +3,24 @@ package controlplane import ( "context" "encoding/json" + "errors" "fmt" "time" "github.com/samcharles93/archie-core/internal/config" pb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1" + "github.com/samcharles93/archie-core/internal/store" ) +// resourceReader is the one read the layering performs: a resource kind's value +// and the version it came from. The gRPC client and the in-process server both +// provide it, so "what the database covers in the running config" has exactly +// one implementation. The offline validate runs the same code as boot, which is +// the only way its verdict can be boot's verdict. +type resourceReader interface { + query(ctx context.Context, kind string, decode func([]byte) error) (int64, error) +} + // RuntimeConfig applies restart-scoped database resources over bootstrap // configuration. Values intentionally absent from a control-plane projection, // such as secret-bearing MCP headers and channel update commands, remain @@ -19,9 +30,70 @@ import ( // layers them in can report which version it is running // (docs/prds/control-plane-apply-status.md). func (c *Client) RuntimeConfig(ctx context.Context, base config.Config) (config.Config, map[string]int64, error) { + return runtimeConfigFrom(ctx, c, base) +} + +// RuntimeChatConfig layers the stored channel settings over the file document's +// chat section. +func (c *Client) RuntimeChatConfig(ctx context.Context, base config.ChatConfig) (config.ChatConfig, int64, error) { + return runtimeChatConfigFrom(ctx, c, base) +} + +// StoredRuntimeConfig is RuntimeConfig's layering over the store this server +// owns, for a caller holding the database file rather than a connection to it. +// Boot layers the stored settings onto the file config and then runs +// configuration.Validate, so an offline check of that database has to make the +// same first move with the same implementation; called with the config the +// daemon would load, the result is the document boot decides on. +// +// A kind the store does not hold is read as the seed the State Store would +// write for it from base (see storeReader.query), because that store seeds +// every kind before it serves. +func (s *Server) StoredRuntimeConfig(ctx context.Context, base config.Config) (config.Config, map[string]int64, error) { + seeds, err := s.seededValues(base) + if err != nil { + return config.Config{}, nil, err + } + return runtimeConfigFrom(ctx, storeReader{resources: s.store, seeds: seeds}, base) +} + +// storeReader reads the server's own store. It mirrors what the gRPC read +// answers -- a kind that is stored is the value and version it holds -- with +// one difference: a kind the store does not hold yet is the value the State +// Store seeds for it, since the daemon dials a store that has already run +// ImportConfig over the same config. +type storeReader struct { + resources ResourceStore + seeds map[string][]byte +} + +func (r storeReader) query(ctx context.Context, kind string, decode func([]byte) error) (int64, error) { + resource, err := r.resources.Resource(ctx, kind) + if errors.Is(err, store.ErrResourceNotFound) { + seed, ok := r.seeds[kind] + if !ok { + return 0, fmt.Errorf("read %s: %w", kind, store.ErrResourceNotFound) + } + if err := decode(seed); err != nil { + return 0, fmt.Errorf("decode %s seed: %w", kind, err) + } + // No version: nothing has been stored for this kind, so nothing has + // been applied either. + return 0, nil + } + if err != nil { + return 0, fmt.Errorf("read %s: %w", kind, err) + } + if err := decode(resource.Value); err != nil { + return 0, fmt.Errorf("decode %s: %w", kind, err) + } + return resource.Version, nil +} + +func runtimeConfigFrom(ctx context.Context, reader resourceReader, base config.Config) (config.Config, map[string]int64, error) { out := base.Clone() versions := map[string]int64{} - if err := c.queryInto(ctx, versions, ProviderSettingsKind, func(value []byte) error { + if err := layerResource(ctx, reader, versions, ProviderSettingsKind, func(value []byte) error { var providers map[string]providerDocument if err := json.Unmarshal(value, &providers); err != nil { return err @@ -34,18 +106,18 @@ func (c *Client) RuntimeConfig(ctx context.Context, base config.Config) (config. }); err != nil { return config.Config{}, nil, err } - if err := c.queryJSONInto(ctx, versions, ModelRoleAssignmentsKind, &out.Models); err != nil { + if err := layerResourceJSON(ctx, reader, versions, ModelRoleAssignmentsKind, &out.Models); err != nil { return config.Config{}, nil, err } - if err := c.queryJSONInto(ctx, versions, RepositoryPoliciesKind, &out.Repos); err != nil { + if err := layerResourceJSON(ctx, reader, versions, RepositoryPoliciesKind, &out.Repos); err != nil { return config.Config{}, nil, err } - chat, chatVersion, err := c.RuntimeChatConfig(ctx, out.Chat) + chat, chatVersion, err := runtimeChatConfigFrom(ctx, reader, out.Chat) if err != nil { return config.Config{}, nil, err } out.Chat, versions[ChannelSettingsKind] = chat, chatVersion - if err := c.queryInto(ctx, versions, SchedulingPolicyKind, func(value []byte) error { + if err := layerResource(ctx, reader, versions, SchedulingPolicyKind, func(value []byte) error { var policy schedulingPolicy if err := json.Unmarshal(value, &policy); err != nil { return err @@ -59,11 +131,11 @@ func (c *Client) RuntimeConfig(ctx context.Context, base config.Config) (config. }); err != nil { return config.Config{}, nil, err } - return c.runtimeToolConfig(ctx, versions, out) + return runtimeToolConfigFrom(ctx, reader, versions, out) } -func (c *Client) runtimeToolConfig(ctx context.Context, versions map[string]int64, out config.Config) (config.Config, map[string]int64, error) { - if err := c.queryInto(ctx, versions, ToolSettingsKind, func(value []byte) error { +func runtimeToolConfigFrom(ctx context.Context, reader resourceReader, versions map[string]int64, out config.Config) (config.Config, map[string]int64, error) { + if err := layerResource(ctx, reader, versions, ToolSettingsKind, func(value []byte) error { var settings toolSettings if err := json.Unmarshal(value, &settings); err != nil { return err @@ -81,7 +153,7 @@ func (c *Client) runtimeToolConfig(ctx context.Context, versions map[string]int6 }); err != nil { return config.Config{}, nil, err } - if err := c.queryInto(ctx, versions, PluginSettingsKind, func(value []byte) error { + if err := layerResource(ctx, reader, versions, PluginSettingsKind, func(value []byte) error { var settings pluginSettings if err := json.Unmarshal(value, &settings); err != nil { return err @@ -91,15 +163,15 @@ func (c *Client) runtimeToolConfig(ctx context.Context, versions map[string]int6 }); err != nil { return config.Config{}, nil, err } - if err := c.queryJSONInto(ctx, versions, ContainerRuntimePoliciesKind, &out.Containers); err != nil { + if err := layerResourceJSON(ctx, reader, versions, ContainerRuntimePoliciesKind, &out.Containers); err != nil { return config.Config{}, nil, err } return out, versions, nil } -func (c *Client) RuntimeChatConfig(ctx context.Context, base config.ChatConfig) (config.ChatConfig, int64, error) { +func runtimeChatConfigFrom(ctx context.Context, reader resourceReader, base config.ChatConfig) (config.ChatConfig, int64, error) { out := base - version, err := c.query(ctx, ChannelSettingsKind, func(value []byte) error { + version, err := reader.query(ctx, ChannelSettingsKind, func(value []byte) error { var settings channelSettings if err := json.Unmarshal(value, &settings); err != nil { return err @@ -117,10 +189,10 @@ func (c *Client) RuntimeChatConfig(ctx context.Context, base config.ChatConfig) return out, version, err } -// queryInto decodes a resource and records the version it came from, so -// RuntimeConfig ends up holding the version of every kind it layered in. -func (c *Client) queryInto(ctx context.Context, versions map[string]int64, kind string, decode func([]byte) error) error { - version, err := c.query(ctx, kind, decode) +// layerResource decodes a resource and records the version it came from, so the +// layering ends up holding the version of every kind it applied. +func layerResource(ctx context.Context, reader resourceReader, versions map[string]int64, kind string, decode func([]byte) error) error { + version, err := reader.query(ctx, kind, decode) if err != nil { return err } @@ -128,8 +200,8 @@ func (c *Client) queryInto(ctx context.Context, versions map[string]int64, kind return nil } -func (c *Client) queryJSONInto(ctx context.Context, versions map[string]int64, kind string, target any) error { - return c.queryInto(ctx, versions, kind, func(value []byte) error { return json.Unmarshal(value, target) }) +func layerResourceJSON(ctx context.Context, reader resourceReader, versions map[string]int64, kind string, target any) error { + return layerResource(ctx, reader, versions, kind, func(value []byte) error { return json.Unmarshal(value, target) }) } // query returns the version of the resource it decoded, so callers that layer diff --git a/internal/releaseupdate/install_script_test.go b/internal/releaseupdate/install_script_test.go index fdae8fac4..77f58e27e 100644 --- a/internal/releaseupdate/install_script_test.go +++ b/internal/releaseupdate/install_script_test.go @@ -382,6 +382,13 @@ esac `) writeFakeCommand(t, fakeDir, "go", ` printf '%s\n' "go $*" >> "$ARCHIE_TEST_CALLS" +# The pre-flight task-database backup runs through the release's own +# archie-state-store subcommand, so a release whose recovery path cannot run +# must refuse the update. +if [ "$1" = "run" ]; then + [ "${ARCHIE_TEST_GO_FAILS_RUN:-}" = "1" ] && exit 1 + exit 0 +fi out="" while [ "$#" -gt 0 ]; do if [ "$1" = "-o" ]; then out="$2"; break; fi @@ -745,6 +752,10 @@ func TestUpdateWatchdogCyclesEveryUnitInDependencyOrder(t *testing.T) { // value up produces an empty backup, and a rollback then restores that empty // file over the database and deletes its -wal/-shm. A rollback that destroys // the task store is worse than no rollback. +// +// The snapshot is taken by the release's own recovery subcommand rather than +// by hand-rolled shell, so the installer and an operator run the same code and +// the two cannot drift apart. func TestUpdateInstallBacksUpTheSiblingTaskDatabase(t *testing.T) { work := t.TempDir() configured := filepath.Join(work, "archie.db") @@ -764,7 +775,8 @@ func TestUpdateInstallBacksUpTheSiblingTaskDatabase(t *testing.T) { "ARCHIE_TASK_DB_BACKUP": backup, }, "db_path = \""+configured+"\"\n\n[containers]\nimage = 'registry.example/archie-agent:stable'\n", false) - assertCallContains(t, calls, "sqlite3", realStore, ".backup") + assertCallContains(t, calls, "go run ./cmd/archie-state-store backup", "-db "+realStore, "-out "+backup) + assertCallAbsent(t, calls, "sqlite3") assertCallContains(t, calls, "systemd-run", "--setenv=ARCHIE_TASK_DB_PATH="+realStore) // The zero-byte placeholder is not the store, and must never be the thing // handed to the watchdog as ARCHIE_TASK_DB_PATH. @@ -793,9 +805,9 @@ func TestUpdateInstallIgnoresDbPathFromOtherSections(t *testing.T) { "ARCHIE_TASK_DB_BACKUP": backup, }, "db_path = \""+configured+"\"\n\n[indexing]\ndb_path = \""+indexStore+"\"\n\n[containers]\nimage = 'registry.example/archie-agent:stable'\n", false) - assertCallContains(t, calls, "sqlite3", realStore) + assertCallContains(t, calls, "go run ./cmd/archie-state-store backup", "-db "+realStore) for _, call := range calls { - if strings.Contains(call, "sqlite3") && strings.Contains(call, indexStore) { + if strings.Contains(call, "go run ./cmd/archie-state-store backup") && strings.Contains(call, indexStore) { t.Errorf("backed up the indexing database %s: %q", indexStore, call) } } @@ -815,7 +827,7 @@ func TestUpdateInstallRefusesWhenTaskDatabaseCannotBeBackedUp(t *testing.T) { "ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0", "ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0", "ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0", - "ARCHIE_TEST_SQLITE3_FAILS": "1", + "ARCHIE_TEST_GO_FAILS_RUN": "1", }, "db_path = \""+configured+"\"\n\n[containers]\nimage = 'registry.example/archie-agent:stable'\n", true) if !strings.Contains(output, "task database") { diff --git a/internal/store/maintenance.go b/internal/store/maintenance.go new file mode 100644 index 000000000..d1165a600 --- /dev/null +++ b/internal/store/maintenance.go @@ -0,0 +1,265 @@ +package store + +import ( + "context" + "database/sql" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" +) + +// This file is the offline half of the store's contract with its owner: the +// operations the State Store process cannot perform on itself, because they +// read or replace the very file it has open. They are what +// docs/prds/runtime-control-plane.md ("Bootstrap, migration, and recovery") +// asks for -- back up, restore and validate the database without archied or +// the Web UI -- and every one of them works on the file directly. +// +// The read-only operations never take the ownership lock, because they are +// what an operator runs against a serving store. The one that replaces the +// file does, because a rewrite under a running server is the only way an +// offline command can destroy a task store. + +// OpenReadOnly opens an existing store for reading only: no schema is created +// and no migration runs. That matters for recovery -- the snapshot an update +// takes has to stay exactly what the release that wrote it expects, and +// migrating a copy during verification would hand a rollback a schema its +// binary refuses. +func OpenReadOnly(ctx context.Context, path string) (*Store, error) { + if err := RequireDatabase(path); err != nil { + return nil, err + } + db, err := sql.Open("sqlite", readonlyDSN(path)) + if err != nil { + return nil, err + } + db.SetMaxOpenConns(1) + // sql.Open is lazy: the first statement is where a file that is not a + // database, a missing WAL sidecar or an unreadable path surfaces. + var tables int + if err := db.QueryRowContext(ctx, `SELECT count(*) FROM sqlite_master`).Scan(&tables); err != nil { + return nil, errors.Join(fmt.Errorf("read task database %s: %w", path, err), db.Close()) + } + return &Store{db: db}, nil +} + +// readonlyDSN reads the store without the write-ahead log pragma the serving +// process sets: a reader has no business changing the database's journal mode, +// and the read-only flag makes an accidental write an error rather than a +// silent modification of a file someone else owns. +func readonlyDSN(path string) string { + separator := "?" + if strings.HasSuffix(path, "?") || strings.HasSuffix(path, "&") { + separator = "" + } else if strings.Contains(path, "?") { + separator = "&" + } + return path + separator + "mode=ro&_pragma=busy_timeout(5000)" +} + +// StoresResources reports whether the store's file carries the control-plane +// resources table. A store written before the control plane existed has none +// and holds no stored settings to validate; the serving process creates the +// table on its next start, so an offline check must tell that apart from +// resources it cannot read. +func (s *Store) StoresResources(ctx context.Context) (bool, error) { + var tables int + if err := s.db.QueryRowContext(ctx, `SELECT count(*) FROM sqlite_master WHERE type='table' AND name='resources'`).Scan(&tables); err != nil { + return false, fmt.Errorf("inspect store schema: %w", err) + } + return tables > 0, nil +} + +// RequireDatabase reports whether path is an existing store file. A writer +// that must not create the file it fails to find asks this first: a recovery +// command pointed at a mistyped path has to refuse, not leave a fresh empty +// database where an operator expected theirs. +func RequireDatabase(path string) error { + if _, err := os.Stat(path); err != nil { + return fmt.Errorf("task database %s: %w", path, err) + } + return nil +} + +// ValidateFile checks an existing store the way the serving process meets it +// on startup and reports the schema version it carries: the file is a SQLite +// database this binary can read, it is not corrupt, and its schema is not +// newer than this binary supports. +func ValidateFile(ctx context.Context, path string) (int, error) { + st, err := OpenReadOnly(ctx, path) + if err != nil { + return 0, err + } + defer func() { _ = st.Close() }() + return st.check(ctx) +} + +// check runs the file-level checks. integrity_check returns one row per +// problem, or the single row "ok"; reading the first row is therefore the +// verdict rather than a sample. +func (s *Store) check(ctx context.Context) (int, error) { + var result string + if err := s.db.QueryRowContext(ctx, `PRAGMA integrity_check`).Scan(&result); err != nil { + return 0, fmt.Errorf("integrity check: %w", err) + } + if result != "ok" { + return 0, fmt.Errorf("integrity check: %s", result) + } + var version int + if err := s.db.QueryRowContext(ctx, `PRAGMA user_version`).Scan(&version); err != nil { + return 0, fmt.Errorf("read schema version: %w", err) + } + if version > taskSchemaVersion { + return 0, fmt.Errorf("database schema version %d is newer than supported version %d", version, taskSchemaVersion) + } + return version, nil +} + +// Backup writes a transactionally consistent snapshot of database to snapshot, +// using SQLite's own copy, so a store that is being written to cannot produce +// a torn file. +// +// It deliberately does not take the ownership lock. The update installer runs +// inside the process it is updating and cannot stop the State Store, so the +// snapshot that makes a failed update reversible is taken against a serving +// store; a backup that insisted on an exclusively owned file would break the +// one caller that exists. The snapshot is written beside the destination and +// renamed into place, so an interrupted backup never replaces a snapshot that +// was good with one that is not. +func Backup(ctx context.Context, database, snapshot string) error { + if err := RequireDatabase(database); err != nil { + return err + } + // The exemption from the ownership lock below is only sound while backup + // never replaces the database. With -out naming the database it would + // rename the snapshot over a file a serving State Store may still have + // open, orphaning the WAL of the file it just unlinked -- so it is refused + // exactly as Restore refuses it. + if same, err := sameFile(database, snapshot); err != nil { + return err + } else if same { + return fmt.Errorf("snapshot %s is the database itself", snapshot) + } + if err := os.MkdirAll(filepath.Dir(snapshot), 0o755); err != nil { + return fmt.Errorf("create snapshot directory: %w", err) + } + db, err := sql.Open("sqlite", sqliteDSN(database)) + if err != nil { + return fmt.Errorf("open task database %s: %w", database, err) + } + defer func() { _ = db.Close() }() + + staging := snapshot + ".tmp" + if err := os.Remove(staging); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("clear staging snapshot %s: %w", staging, err) + } + if _, err := db.ExecContext(ctx, `VACUUM INTO ?`, staging); err != nil { + return errors.Join(fmt.Errorf("snapshot %s: %w", database, err), os.Remove(staging)) + } + // A snapshot that is not itself a usable store is worse than no snapshot: + // the update path treats its existence as proof that a rollback is + // possible. + if _, err := ValidateFile(ctx, staging); err != nil { + return errors.Join(fmt.Errorf("snapshot is not a usable store: %w", err), os.Remove(staging)) + } + if err := os.Rename(staging, snapshot); err != nil { + return errors.Join(fmt.Errorf("publish snapshot %s: %w", snapshot, err), os.Remove(staging)) + } + return nil +} + +// Restore replaces database with the snapshot at snapshot. The snapshot is +// verified before anything is destroyed, so a file that is not a usable store +// cannot take the database down with it, and the database file itself is +// replaced by a rename, so a failure leaves either the old file or the new one +// rather than a mixture. The replaced file's -wal/-shm are removed first, which +// the comment at that step explains: between the two calls the database is +// merely incomplete, and re-running the command completes it. +// +// It holds the ownership lock for its duration: rewriting a database a running +// State Store still has open would leave that process writing to an unlinked +// inode while the store served a file nobody owns. +func Restore(ctx context.Context, database, snapshot string) error { + if _, err := os.Stat(filepath.Dir(database)); err != nil { + return fmt.Errorf("restore destination %s: %w", database, err) + } + if same, err := sameFile(database, snapshot); err != nil { + return err + } else if same { + return fmt.Errorf("restore snapshot %s is the database itself", snapshot) + } + ownership, err := AcquireOwnership(database) + if err != nil { + return err + } + defer func() { _ = ownership.Release() }() + + if _, err := ValidateFile(ctx, snapshot); err != nil { + return fmt.Errorf("snapshot is not a usable store: %w", err) + } + staging := database + ".restore.tmp" + if err := copyFile(snapshot, staging); err != nil { + return errors.Join(err, os.Remove(staging)) + } // The replaced database's write-ahead log belongs to the file that is + // going away. It goes first: between the two calls the database is merely + // incomplete, whereas a WAL left beside the file it does not describe is + // the state SQLite cannot recover from. + for _, sidecar := range []string{"-wal", "-shm"} { + if err := os.Remove(database + sidecar); err != nil && !os.IsNotExist(err) { + return errors.Join(fmt.Errorf("remove %s%s: %w", database, sidecar, err), os.Remove(staging)) + } + } + if err := os.Rename(staging, database); err != nil { + return errors.Join(fmt.Errorf("publish restored database %s: %w", database, err), os.Remove(staging)) + } + return nil +} + +func sameFile(a, b string) (bool, error) { + first, err := os.Stat(a) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + second, err := os.Stat(b) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + return os.SameFile(first, second), nil +} + +// copyFile writes source over a new file at path, flushed to disk before it is +// returned: the rename that publishes it must not reach the directory entries +// ahead of the bytes they point at. The snapshot's own permissions are kept, so +// a restore does not silently change who may read the operator's database. +func copyFile(source, path string) (retErr error) { + info, err := os.Stat(source) + if err != nil { + return fmt.Errorf("stat snapshot %s: %w", source, err) + } + in, err := os.Open(source) + if err != nil { + return fmt.Errorf("open snapshot %s: %w", source, err) + } + defer func() { retErr = errors.Join(retErr, in.Close()) }() + out, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, info.Mode().Perm()) + if err != nil { + return fmt.Errorf("create %s: %w", path, err) + } + defer func() { retErr = errors.Join(retErr, out.Close()) }() + if _, err := io.Copy(out, in); err != nil { + return fmt.Errorf("write %s: %w", path, err) + } + if err := out.Sync(); err != nil { + return fmt.Errorf("sync %s: %w", path, err) + } + return nil +} diff --git a/internal/store/maintenance_test.go b/internal/store/maintenance_test.go new file mode 100644 index 000000000..95eda0c86 --- /dev/null +++ b/internal/store/maintenance_test.go @@ -0,0 +1,92 @@ +package store + +import ( + "context" + "database/sql" + "errors" + "fmt" + "os" + "path/filepath" + "testing" +) + +// Ownership is what makes "the State Store is stopped" a fact an offline +// command can check, so the primitive itself is asserted here rather than only +// through the commands that use it: exclusivity, release, and per-database +// independence. +func TestOwnershipIsExclusivePerDatabase(t *testing.T) { + dir := t.TempDir() + first := filepath.Join(dir, "archie.db-tasks.sqlite") + second := filepath.Join(dir, "other.db-tasks.sqlite") + + held, err := AcquireOwnership(first) + if err != nil { + t.Fatalf("first AcquireOwnership: %v", err) + } + if _, err := AcquireOwnership(first); !errors.Is(err, ErrStoreOwned) { + t.Fatalf("second AcquireOwnership on the same database = %v, want ErrStoreOwned", err) + } + // A different database is a different claim: one process owning its store + // must not stop another from owning its own. + elsewhere, err := AcquireOwnership(second) + if err != nil { + t.Fatalf("AcquireOwnership on an unrelated database: %v", err) + } + if _, err := os.Stat(first + ".lock"); err != nil { + t.Fatalf("ownership left no sidecar to lock: %v", err) + } + + if err := held.Release(); err != nil { + t.Fatalf("Release: %v", err) + } + reacquired, err := AcquireOwnership(first) + if err != nil { + t.Fatalf("AcquireOwnership after Release: %v", err) + } + if err := errors.Join(reacquired.Release(), elsewhere.Release()); err != nil { + t.Fatalf("release on cleanup: %v", err) + } +} + +// The snapshot an update takes is the state a rollback restores, and it is +// restored by the release that wrote it. Verifying the copy must therefore not +// migrate it: a snapshot silently upgraded to this binary's schema would be +// refused by the older binary it exists to serve. +func TestBackupSnapshotKeepsTheSchemaVersionItCopied(t *testing.T) { + dir := t.TempDir() + database := filepath.Join(dir, "archie.db-tasks.sqlite") + snapshot := filepath.Join(dir, "snapshot.sqlite") + st, err := Open(t.Context(), database) + if err != nil { + t.Fatal(err) + } + if err := st.Close(); err != nil { + t.Fatal(err) + } + // An older release's file: the schema version it wrote, which this binary + // is expected to migrate when the store is next opened for serving. + if err := setUserVersion(t.Context(), database, taskSchemaVersion-1); err != nil { + t.Fatal(err) + } + + if err := Backup(t.Context(), database, snapshot); err != nil { + t.Fatalf("Backup: %v", err) + } + version, err := ValidateFile(t.Context(), snapshot) + if err != nil { + t.Fatalf("ValidateFile on the snapshot: %v", err) + } + if version != taskSchemaVersion-1 { + t.Fatalf("snapshot schema version = %d, want the %d it copied", version, taskSchemaVersion-1) + } +} + +func setUserVersion(ctx context.Context, path string, version int) error { + db, err := sql.Open("sqlite", path) + if err != nil { + return err + } + defer func() { _ = db.Close() }() + _, err = db.ExecContext(ctx, fmt.Sprintf(`PRAGMA user_version = %d`, version)) + return err +} diff --git a/internal/store/ownership.go b/internal/store/ownership.go new file mode 100644 index 000000000..b0743d89f --- /dev/null +++ b/internal/store/ownership.go @@ -0,0 +1,68 @@ +package store + +import ( + "errors" + "fmt" + "os" + "path/filepath" +) + +// ErrStoreOwned is returned when a task database is already owned by another +// process. Ownership is what makes "the State Store is stopped" a fact an +// offline command can check rather than a rule the operator is trusted to +// follow: overwriting a database a running server still holds leaves it +// writing to an unlinked inode while the store serves a file nobody owns. +var ErrStoreOwned = errors.New("task database is owned by another process") + +// Ownership is one process's exclusive claim over one task database, held for +// as long as that process owns the file. The serving State Store takes it for +// its whole life; the offline recovery commands take it for the duration of an +// operation that rewrites the file. +// +// The claim lives in a sidecar file beside the database rather than on the +// database inode itself, so it survives the rename a restore performs: after +// the swap the sidecar is still the one path every process locks. +type Ownership struct { + f *os.File +} + +// AcquireOwnership takes the store's ownership lock without blocking. A +// database another process owns comes back as ErrStoreOwned, never as a wait: +// a process that must not proceed has to hear so, and a blocking lock would +// turn a mistyped command into a hung terminal. +func AcquireOwnership(path string) (*Ownership, error) { + lockPath := ownershipLockPath(path) + if err := os.MkdirAll(filepath.Dir(lockPath), 0o755); err != nil { + return nil, fmt.Errorf("create store directory for %s: %w", lockPath, err) + } + f, err := os.OpenFile(lockPath, os.O_RDWR|os.O_CREATE, 0o644) + if err != nil { + return nil, fmt.Errorf("open store lock %s: %w", lockPath, err) + } + taken, err := lockFile(f) + if err != nil { + return nil, errors.Join(fmt.Errorf("lock %s: %w", lockPath, err), f.Close()) + } + if !taken { + return nil, errors.Join(fmt.Errorf("%s: %w", path, ErrStoreOwned), f.Close()) + } + return &Ownership{f: f}, nil +} + +// Release drops the claim and closes the sidecar. Closing the descriptor alone +// would release it too; the explicit unlock keeps the intent readable and +// makes Release idempotent. +func (o *Ownership) Release() error { + if o == nil || o.f == nil { + return nil + } + err := unlockFile(o.f) + f := o.f + o.f = nil + return errors.Join(err, f.Close()) +} + +// ownershipLockPath is the sidecar every owner of path locks. It is derived +// from the database path, so the State Store process and an offline command +// that were given the same database agree on it without configuration. +func ownershipLockPath(path string) string { return path + ".lock" } diff --git a/internal/store/ownership_unix.go b/internal/store/ownership_unix.go new file mode 100644 index 000000000..3d0d2bb4b --- /dev/null +++ b/internal/store/ownership_unix.go @@ -0,0 +1,30 @@ +//go:build unix + +package store + +import ( + "errors" + "os" + "syscall" +) + +// lockFile takes a non-blocking exclusive flock on f. The boolean reports +// whether it was taken: a lock another process holds (EWOULDBLOCK) is a normal +// answer to "is the State Store running?", not an error. +func lockFile(f *os.File) (bool, error) { + err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) + if errors.Is(err, syscall.EWOULDBLOCK) { + return false, nil + } + if err != nil { + return false, err + } + return true, nil +} + +func unlockFile(f *os.File) error { + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_UN); err != nil { + return err + } + return nil +} diff --git a/internal/store/ownership_windows.go b/internal/store/ownership_windows.go new file mode 100644 index 000000000..e282ae29d --- /dev/null +++ b/internal/store/ownership_windows.go @@ -0,0 +1,23 @@ +//go:build windows + +// flock is a POSIX primitive and Windows is not a supported deployment surface +// (CLAUDE.md's deployment model is Linux and macOS), so this file fails closed: +// every lock attempt reports an error rather than silently degrading to "no +// lock". An offline command that cannot prove it owns the database must refuse +// to rewrite it, which is the only safe reading of an unavailable primitive. +// +// If a Windows deployment ever becomes supported, swap this for a real +// LockFileEx/UnlockFileEx pair; nothing else in the package needs to change. + +package store + +import ( + "errors" + "os" +) + +func lockFile(*os.File) (bool, error) { + return false, errors.New("store ownership locking requires a POSIX flock, which this platform does not provide") +} + +func unlockFile(*os.File) error { return nil } diff --git a/scripts/archie-update-install b/scripts/archie-update-install index ce3635f63..30c43bef4 100755 --- a/scripts/archie-update-install +++ b/scripts/archie-update-install @@ -183,10 +183,18 @@ fi # that cannot take one must not start: once binaries are replaced, a failure # leaves the previous release unable to read the store it just migrated. This # runs ahead of the adapter refresh below so a refusal changes nothing at all. +# +# The snapshot is taken by the recovery subcommand of the release being +# installed, from its own checkout, rather than by hand-rolled sqlite3 shell: +# the operator running `archie-state-store backup` by hand and the installer +# are then the same implementation, and the two cannot drift. The installed +# binary is the previous release, which may predate the subcommand, so it +# cannot be the one asked -- and a release whose recovery path will not build is a +# release that cannot back up, which is exactly what this pre-flight refuses. if [[ ",$components," == *,daemon,* ]] && [ -n "$TASK_DB_PATH" ] && [ -f "$TASK_DB_PATH" ]; then echo "==> backing up task database for schema-safe rollback" mkdir -p "$(dirname "$TASK_DB_BACKUP")" - if ! sqlite3 "$TASK_DB_PATH" ".backup '$TASK_DB_BACKUP'"; then + if ! go run ./cmd/archie-state-store backup -db "$TASK_DB_PATH" -out "$TASK_DB_BACKUP"; then echo "refusing to install $GW: could not back up the task database at $TASK_DB_PATH, so a failed update could not be rolled back" >&2 exit 1 fi