feat(statestore): offline backup, restore, validate and rollback

Resolves archie-core-6zw0. Subcommands of cmd/archie-state-store (not a new
binary, per organisation.md#process-boundaries) operating on the task database
file directly with the State Store stopped: backup, restore, validate, rollback.
A revision carries its value, so restoring replays it through the ordinary
replace and no rollback RPC is added. The pre-flight backup in
scripts/archie-update-install now calls the subcommand; the watchdog's refusal
semantics are unchanged.
This commit is contained in:
sam committed 2026-09-21 22:08:12 +10:00
commit 494580fad6
17 files changed
+1904 -43

No files matched your search

+12
View File
@@ -15,6 +15,7 @@ import (
"os"
"os/signal"
"path/filepath"
"strings"
"syscall"
"github.com/samcharles93/archie-core/internal/app/archied"
@@ -23,6 +24,13 @@ import (
func main() { os.Exit(run()) }
func run() int {
// A leading non-flag argument selects a recovery subcommand; anything else,
// including no arguments at all, serves the State Store contract. An
// unrecognised word is refused rather than ignored, so a mistyped recovery
// command never leaves an operator watching a server start instead.
if args := os.Args[1:]; len(args) > 0 && !strings.HasPrefix(args[0], "-") {
return runRecovery(args, os.Stdout, os.Stderr)
}
base, err := os.UserConfigDir()
if err != nil {
fmt.Fprintln(os.Stderr, err)
@@ -34,6 +42,10 @@ func run() int {
flag.StringVar(&options.Listen, "listen", "", "state store gRPC listen address (defaults to [services.state].listen, else 127.0.0.1:9090)")
flag.StringVar(&options.Token, "token", "", "bearer token required for a non-loopback listener (defaults to [services.state].target_token)")
flag.StringVar(&options.ReadyAddr, "ready-addr", "", "optional readiness HTTP listen address (e.g. 127.0.0.1:9091)")
flag.Usage = func() {
fmt.Fprintf(flag.CommandLine.Output(), "usage: archie-state-store [flags] # serve the State Store gRPC contract\n\noffline recovery: archie-state-store <backup|restore|validate|rollback> -h\n\n")
flag.PrintDefaults()
}
flag.Parse()
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
+94
View File
@@ -0,0 +1,94 @@
package main
import (
"context"
"errors"
"flag"
"fmt"
"io"
"github.com/samcharles93/archie-core/internal/app/archied"
)
// defaultConfigPath is the configuration the serve path reads by default, and
// the one the daemon boots with. It is the daemon's own rule rather than a
// second derivation of it: os.UserConfigDir (what the sibling processes use)
// disagrees with archied on darwin and rejects a relative XDG_CONFIG_HOME
// outright, which would leave validate answering about a file nobody boots.
func defaultConfigPath() string { return archied.DefaultConfigPath() }
// recoveryUsage is the whole flag surface of the offline recovery commands.
// They are subcommands of this binary rather than a recovery binary of their
// own: offline database maintenance belongs to the process that owns the file
// and its path layout (docs/architecture/organisation.md#process-boundaries).
const recoveryUsage = `usage: archie-state-store <command> [flags]
With no command, archie-state-store serves the State Store gRPC contract.
The recovery commands operate on the task database file directly -- the
configured db_path with "-tasks.sqlite" appended -- and are how an operator
recovers a database the control plane will not start on. Stop the State Store
first: restore and rollback rewrite the file and refuse while another process
owns it, while backup takes a consistent snapshot of a serving store.
backup -db FILE -out FILE write a snapshot of the database
restore -db FILE -from FILE replace the database with a snapshot
validate -db FILE [-config FILE] check the file, its stored settings and the
configuration the daemon would boot with
rollback -db FILE -kind KIND replay an earlier revision of a stored
[-revision N] resource through the ordinary replace
`
// runRecovery parses and performs one offline recovery command, reporting the
// operation's summary on stdout. Exit codes follow the serve path: 0 succeeded,
// 1 the operation failed, 2 the command line was wrong.
func runRecovery(args []string, stdout, stderr io.Writer) int {
if len(args) == 0 {
fmt.Fprint(stderr, recoveryUsage)
return 2
}
command := args[0]
options := archied.StateStoreRecoveryOptions{Operation: command}
switch command {
case archied.RecoveryBackup, archied.RecoveryRestore, archied.RecoveryValidate, archied.RecoveryRollback:
default:
fmt.Fprintf(stderr, "archie-state-store: unknown command %q\n\n", command)
fmt.Fprint(stderr, recoveryUsage)
return 2
}
flags := flag.NewFlagSet("archie-state-store "+command, flag.ContinueOnError)
flags.SetOutput(stderr)
flags.Usage = func() { fmt.Fprint(stderr, recoveryUsage) }
flags.StringVar(&options.DB, "db", "", "task database file the State Store owns (<db_path>-tasks.sqlite)")
flags.StringVar(&options.Out, "out", "", "snapshot file backup writes")
flags.StringVar(&options.From, "from", "", "snapshot file restore reads")
flags.StringVar(&options.Kind, "kind", "", "control-plane resource kind rollback replays")
flags.Int64Var(&options.Revision, "revision", 0, "revision rollback replays (default: the newest one older than the current value)")
// Only validate asks a question about the process rather than the file, so
// only validate takes the configuration the process would boot with.
if command == archied.RecoveryValidate {
flags.StringVar(&options.Config, "config", defaultConfigPath(), "configuration file or directory the daemon boots with")
flags.StringVar(&options.Overlay, "config-overlay", "", "configuration overlay file or directory")
}
if err := flags.Parse(args[1:]); err != nil {
// A requested help is not a wrong command line: the usage above is the
// answer, and the serve path exits 0 for the same request.
if errors.Is(err, flag.ErrHelp) {
return 0
}
return 2
}
if flags.NArg() > 0 {
fmt.Fprintf(stderr, "archie-state-store %s: unexpected argument %q\n", command, flags.Arg(0))
return 2
}
summary, err := archied.RunStateStoreRecovery(context.Background(), options)
if err != nil {
fmt.Fprintf(stderr, "archie-state-store %s: %v\n", command, err)
return 1
}
fmt.Fprintln(stdout, summary)
return 0
}
+770
View File
@@ -0,0 +1,770 @@
// Package main tests the offline recovery subcommands of archie-state-store:
// backup, restore, validate and rollback against the task database file
// directly. They are the operator's only path back when the control plane's
// stored settings will not validate, because the daemon fails closed: the
// in-band remedy (replay an earlier revision while the State Store is up) is
// unavailable in exactly the case that needs it.
//
// The tests drive the subcommands the way an operator does -- through
// runRecovery, against real files on disk -- rather than through the internal
// helpers, so the flag surface and the exit codes are part of what is verified.
package main
import (
"bytes"
"database/sql"
"encoding/json"
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"testing"
_ "modernc.org/sqlite"
"github.com/samcharles93/archie-core/internal/app/archied"
"github.com/samcharles93/archie-core/internal/app/controlplane"
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
"github.com/samcharles93/archie-core/internal/infrastructure/workflowsteps"
"github.com/samcharles93/archie-core/internal/store"
)
// runRecoveryCmd runs one subcommand the way main does and returns its exit
// code with both streams captured.
func runRecoveryCmd(t *testing.T, args ...string) (int, string, string) {
t.Helper()
var stdout, stderr bytes.Buffer
code := runRecovery(args, &stdout, &stderr)
return code, stdout.String(), stderr.String()
}
// openTaskStore opens the task database the subcommands operate on. It does not
// take the ownership lock: only the serving process does that, and a test that
// wants a "live" store holds it explicitly with holdStoreOwnership.
func openTaskStore(t *testing.T, path string) *store.Store {
t.Helper()
st, err := store.Open(t.Context(), path)
if err != nil {
t.Fatalf("open task store %s: %v", path, err)
}
t.Cleanup(func() { _ = st.Close() })
return st
}
// holdStoreOwnership takes the ownership lock the running State Store holds for
// its whole life, so a test can prove a command refuses to touch a live store.
func holdStoreOwnership(t *testing.T, path string) {
t.Helper()
ownership, err := store.AcquireOwnership(path)
if err != nil {
t.Fatalf("acquire store ownership: %v", err)
}
t.Cleanup(func() { _ = ownership.Release() })
}
// seedTask writes one task so a snapshot can be distinguished from the live
// database by what it contains, not merely by its existence.
func seedTask(t *testing.T, st *store.Store, title string) {
t.Helper()
if _, err := st.EnqueueChatTask(t.Context(), "acme", "widget", title, "body", "implement", ""); err != nil {
t.Fatalf("seed task %q: %v", title, err)
}
}
func taskTitles(t *testing.T, path string) []string {
t.Helper()
st := openTaskStore(t, path)
tasks, err := st.Tasks(t.Context(), 100)
if err != nil {
t.Fatalf("list tasks in %s: %v", path, err)
}
titles := make([]string, 0, len(tasks))
for _, task := range tasks {
titles = append(titles, task.Title)
}
return titles
}
// backupPath is the snapshot every restore test restores from.
func backupPath(t *testing.T, dir string) string { return filepath.Join(dir, "snapshot.sqlite") }
// backup takes the snapshot the caller then restores, through the subcommand
// under test rather than through the store directly, so a broken backup fails
// the restore test too instead of silently passing an empty file.
func backup(t *testing.T, db, out string) {
t.Helper()
code, _, stderr := runRecoveryCmd(t, "backup", "-db", db, "-out", out)
if code != 0 {
t.Fatalf("backup exited %d: %s", code, stderr)
}
}
// The update installer cannot stop the process that runs it, so it backs the
// task database up while the State Store is serving. VACUUM INTO is safe
// against a live writer; a backup that insisted on an exclusively-owned file
// would break the one caller it has, and the update would refuse to start.
func TestRecoveryBackupSnapshotsAStoreItsStateStoreIsServing(t *testing.T) {
dir := t.TempDir()
db := filepath.Join(dir, "archie.db-tasks.sqlite")
out := backupPath(t, dir)
seedTask(t, openTaskStore(t, db), "before")
holdStoreOwnership(t, db)
if err := os.WriteFile(out, []byte("stale snapshot"), 0o600); err != nil {
t.Fatal(err)
}
code, stdout, stderr := runRecoveryCmd(t, "backup", "-db", db, "-out", out)
if code != 0 {
t.Fatalf("backup exited %d: %s", code, stderr)
}
if !strings.Contains(stdout, out) || !strings.Contains(stdout, db) {
t.Errorf("backup must report the database and the snapshot it wrote; stdout = %q", stdout)
}
// The stale file was replaced by a real snapshot, and the snapshot is a
// store a restore can put back: same rows, readable through the store.
if got := taskTitles(t, out); len(got) != 1 || got[0] != "before" {
t.Fatalf("snapshot tasks = %v, want the one seeded task", got)
}
if entries, err := filepath.Glob(out + ".tmp*"); err != nil || len(entries) != 0 {
t.Errorf("backup left scratch files beside the snapshot: %v (%v)", entries, err)
}
}
// The snapshot is the rollback state, so an interrupted backup must never
// leave a truncated file where a good snapshot was.
func TestRecoveryBackupRefusesAMissingDatabase(t *testing.T) {
dir := t.TempDir()
code, _, stderr := runRecoveryCmd(t, "backup", "-db", filepath.Join(dir, "absent.sqlite"), "-out", backupPath(t, dir))
if code == 0 {
t.Fatal("backup of a missing database succeeded")
}
if !strings.Contains(stderr, "absent.sqlite") {
t.Errorf("refusal must name the database it could not read; stderr = %q", stderr)
}
}
// backup is the one recovery command allowed to run against a serving store,
// and that exemption is only sound while it never replaces the database. With
// -out naming the database it renames the snapshot over the live file and
// leaves the WAL of the database it just unlinked -- the state the restore path
// refuses up front and the code elsewhere calls unrecoverable.
func TestRecoveryBackupRefusesTheDatabaseAsItsOwnSnapshot(t *testing.T) {
db := filepath.Join(t.TempDir(), "archie.db-tasks.sqlite")
st := openTaskStore(t, db)
seedTask(t, st, "live")
code, _, stderr := runRecoveryCmd(t, "backup", "-db", db, "-out", db)
if code == 0 {
t.Fatal("backup over the database itself succeeded")
}
if !strings.Contains(stderr, "is the database itself") {
t.Errorf("refusal must say the snapshot is the database; stderr = %q", stderr)
}
if got := taskTitles(t, db); len(got) != 1 || got[0] != "live" {
t.Fatalf("refused backup changed the database: tasks = %v", got)
}
}
// Restoring is the forward-migration escape hatch: the previous release
// reads this file, so it must be exactly the snapshot, with the WAL that
// belongs to the replaced database gone rather than replayed over it.
func TestRecoveryRestorePutsTheSnapshotBackAndDropsItsWAL(t *testing.T) {
dir := t.TempDir()
db := filepath.Join(dir, "archie.db-tasks.sqlite")
out := backupPath(t, dir)
st := openTaskStore(t, db)
seedTask(t, st, "before")
backup(t, db, out)
seedTask(t, st, "after")
code, stdout, stderr := runRecoveryCmd(t, "restore", "-db", db, "-from", out)
if code != 0 {
t.Fatalf("restore exited %d: %s", code, stderr)
}
if !strings.Contains(stdout, out) {
t.Errorf("restore must report the snapshot it restored from; stdout = %q", stdout)
}
// Checked before anything reopens the database: opening a store sets WAL
// mode, which creates these files, so the point is that the restore did not
// leave the replaced database's log behind -- not that a later reader
// cannot recreate its own.
for _, suffix := range []string{"-wal", "-shm"} {
if _, err := os.Stat(db + suffix); err == nil {
t.Errorf("%s survived the restore: the replaced database's write-ahead log must not be replayed over the snapshot", db+suffix)
}
}
if got := taskTitles(t, db); len(got) != 1 || got[0] != "before" {
t.Fatalf("restored tasks = %v, want only the snapshot's task", got)
}
}
// Overwriting a database a running State Store still owns leaves that process
// writing to an unlinked inode while the store serves a file nobody owns --
// the one way an offline command can destroy a task store.
func TestRecoveryRestoreRefusesAStoreTheStateStoreIsRunning(t *testing.T) {
dir := t.TempDir()
db := filepath.Join(dir, "archie.db-tasks.sqlite")
out := backupPath(t, dir)
st := openTaskStore(t, db)
seedTask(t, st, "before")
backup(t, db, out)
seedTask(t, st, "after")
holdStoreOwnership(t, db)
code, _, stderr := runRecoveryCmd(t, "restore", "-db", db, "-from", out)
if code == 0 {
t.Fatal("restore against a live State Store succeeded")
}
if !strings.Contains(stderr, "owned by another process") {
t.Errorf("refusal must say the database is owned by another process; stderr = %q", stderr)
}
if got := taskTitles(t, db); len(got) != 2 {
t.Fatalf("refused restore changed the database: tasks = %v", got)
}
}
func TestRecoveryRestoreRefusesAFileThatIsNotAStore(t *testing.T) {
dir := t.TempDir()
db := filepath.Join(dir, "archie.db-tasks.sqlite")
seedTask(t, openTaskStore(t, db), "live")
notAStore := filepath.Join(dir, "notes.txt")
if err := os.WriteFile(notAStore, []byte("this is not a database"), 0o600); err != nil {
t.Fatal(err)
}
code, _, stderr := runRecoveryCmd(t, "restore", "-db", db, "-from", notAStore)
if code == 0 {
t.Fatal("restore from a file that is not a store succeeded")
}
if !strings.Contains(stderr, notAStore) {
t.Errorf("refusal must name the snapshot it could not read; stderr = %q", stderr)
}
if got := taskTitles(t, db); len(got) != 1 {
t.Fatalf("refused restore changed the database: tasks = %v", got)
}
}
func TestRecoveryRestoreRefusesTheDatabaseAsItsOwnSnapshot(t *testing.T) {
// Restoring a database from itself would delete the write-ahead log of the
// file it is about to copy, so the one case that must never reach the swap
// is refused up front.
db := filepath.Join(t.TempDir(), "archie.db-tasks.sqlite")
seedTask(t, openTaskStore(t, db), "live")
code, _, stderr := runRecoveryCmd(t, "restore", "-db", db, "-from", db)
if code == 0 {
t.Fatal("restore from the database itself succeeded")
}
if !strings.Contains(stderr, "is the database itself") {
t.Errorf("refusal must say the snapshot is the database; stderr = %q", stderr)
}
if got := taskTitles(t, db); len(got) != 1 {
t.Fatalf("refused restore changed the database: tasks = %v", got)
}
}
func TestRecoveryValidateAcceptsAHealthyStore(t *testing.T) {
dir := t.TempDir()
configPath := writeMinimalConfig(t, dir)
db := filepath.Join(dir, "archie.db-tasks.sqlite")
st := openTaskStore(t, db)
seedTask(t, st, "healthy")
seedStoreResources(t, st, configPath)
code, stdout, stderr := runRecoveryCmd(t, "validate", "-db", db, "-config", configPath)
if code != 0 {
t.Fatalf("validate exited %d: %s", code, stderr)
}
if !strings.Contains(stdout, "schema version") {
t.Errorf("validate must report the schema version it verified; stdout = %q", stdout)
}
// The count is part of what the operator reads, and a check that reports
// "0 stored resources" for a store full of them is worse than no check.
if count := storedResourceCount(t, stdout); count == 0 {
t.Errorf("validate reported no stored resources for a seeded store; stdout = %q", stdout)
}
}
// validate's verdict is only useful if it is the daemon's verdict. Boot refuses
// on configuration.Validate over the file config with every stored resource
// layered onto it (internal/app/archied/control_plane.go), not on the write
// path's own per-resource decode, and the two disagree: the write path accepts
// a scheduling policy boot rejects.
func TestRecoveryValidateRunsTheGateBootRuns(t *testing.T) {
dir := t.TempDir()
configPath := writeMinimalConfig(t, dir)
db := filepath.Join(dir, "archie.db-tasks.sqlite")
seedStoreResources(t, openTaskStore(t, db), configPath)
// The control: a store seeded from this config is one the daemon boots on.
code, _, stderr := runRecoveryCmd(t, "validate", "-db", db, "-config", configPath)
if code != 0 {
t.Fatalf("validate exited %d on a store the daemon starts on: %s", code, stderr)
}
// A hand-edited value, or an older revision of one: the write path's own
// validator never inspects dispatch.trigger, so only the config gate
// catches it, and the daemon exits 1 on it (validateDispatch).
if err := execRaw(t, db, `UPDATE resources SET value='{"poll_interval":"1m","max_retries":3,"dispatch":{"trigger":"bogus"}}' WHERE kind='scheduling-policy'`); err != nil {
t.Fatal(err)
}
code, stdout, stderr := runRecoveryCmd(t, "validate", "-db", db, "-config", configPath)
if code == 0 {
t.Fatalf("validate accepted a stored policy the daemon refuses to boot with; stdout = %q", stdout)
}
if !strings.Contains(stderr, "dispatch.trigger") {
t.Errorf("refusal must name what boot rejects; stderr = %q", stderr)
}
// A store the State Store never seeded holds no kinds at all. Its next start
// creates the resources table and seeds every kind from this same config
// (internal/app/archied/state_store.go), so archied layers those seeds and
// starts: refusing the file here would send an operator to restore a
// snapshot for a store the daemon boots on perfectly well.
unseeded := filepath.Join(dir, "unseeded.db-tasks.sqlite")
openTaskStore(t, unseeded)
code, stdout, stderr = runRecoveryCmd(t, "validate", "-db", unseeded, "-config", configPath)
if code != 0 {
t.Fatalf("validate refused a store the State Store would seed on its next start; stderr = %q", stderr)
}
if !strings.Contains(stdout, "0 stored resources validate") {
t.Errorf("validate must report that there was nothing stored to check; stdout = %q", stdout)
}
// The upgrade case: a release defines a kind the store does not hold yet,
// and validate runs before the State Store has started again. The State
// Store seeds the kind it is missing, so this is the same stance as the
// store with nothing stored at all -- the file is not the reason the daemon
// would refuse to start, and saying otherwise costs the operator a snapshot.
partial := filepath.Join(dir, "partial.db-tasks.sqlite")
seedStoreResources(t, openTaskStore(t, partial), configPath)
if err := execRaw(t, partial, `DELETE FROM resources WHERE kind='provider-settings'`); err != nil {
t.Fatal(err)
}
code, stdout, stderr = runRecoveryCmd(t, "validate", "-db", partial, "-config", configPath)
if code != 0 {
t.Fatalf("validate refused a store missing a kind the State Store seeds; stderr = %q", stderr)
}
if count := storedResourceCount(t, stdout); count == 0 {
t.Errorf("validate must still check the kinds the store does hold; stdout = %q", stdout)
}
}
// The subcommands advertise -h as the way to read their flags, and the serve
// path in the same binary exits 0 for the same request, so a script probing the
// recovery surface must not read a requested help as a failure.
func TestRecoveryHelpExitsZero(t *testing.T) {
commands := []string{archied.RecoveryBackup, archied.RecoveryRestore, archied.RecoveryValidate, archied.RecoveryRollback}
for _, command := range commands {
code, _, stderr := runRecoveryCmd(t, command, "-h")
if code != 0 {
t.Errorf("%s -h exited %d, want 0; stderr = %q", command, code, stderr)
}
}
}
// validate diagnoses a deployment; it is not the deployment. Resolving the boot
// config must not create, append to, or rotate the daemon's log file or the
// directory it lives in: a line stamped component="daemon" in archied's log is
// indistinguishable from the daemon having written it.
func TestRecoveryValidateLeavesTheDaemonLogAlone(t *testing.T) {
dir := t.TempDir()
configPath, logFile := writeConfigWithLogFile(t, dir)
db := filepath.Join(dir, "archie.db-tasks.sqlite")
seedStoreResources(t, openTaskStore(t, db), configPath)
code, _, stderr := runRecoveryCmd(t, "validate", "-db", db, "-config", configPath)
if code != 0 {
t.Fatalf("validate exited %d: %s", code, stderr)
}
if _, err := os.Stat(logFile); !os.IsNotExist(err) {
t.Fatalf("validate touched the daemon's log file %s (stat err = %v)", logFile, err)
}
if _, err := os.Stat(filepath.Dir(logFile)); !os.IsNotExist(err) {
t.Errorf("validate created the log directory %s (stat err = %v)", filepath.Dir(logFile), err)
}
}
// validate's verdict is the daemon's verdict only if it reads the configuration
// the daemon reads. archied resolves that default from its own configHome, which
// honours a relative XDG_CONFIG_HOME; os.UserConfigDir rejects a relative one
// outright, which left the command with no config and no verdict at all.
func TestRecoveryDefaultConfigFollowsTheDaemonsConfigHome(t *testing.T) {
relative := filepath.Join("relative", "config")
t.Setenv("XDG_CONFIG_HOME", relative)
if got, want := defaultConfigPath(), filepath.Join(relative, "archie", "config.toml"); got != want {
t.Errorf("defaultConfigPath() with a relative XDG_CONFIG_HOME = %q, want %q", got, want)
}
absolute := t.TempDir()
t.Setenv("XDG_CONFIG_HOME", absolute)
if got, want := defaultConfigPath(), filepath.Join(absolute, "archie", "config.toml"); got != want {
t.Errorf("defaultConfigPath() = %q, want %q", got, want)
}
// The equality itself: one helper, so the two cannot drift apart again.
if got, want := defaultConfigPath(), archied.DefaultConfigPath(); got != want {
t.Errorf("defaultConfigPath() = %q, but the daemon resolves %q", got, want)
}
}
// validate exists to answer "would archied start against this file". Each case
// below is a state the daemon fails closed on, or a file it cannot read at all.
func TestRecoveryValidateRefusesWhatTheDaemonRefuses(t *testing.T) {
newStore := func(t *testing.T) string {
t.Helper()
db := filepath.Join(t.TempDir(), "archie.db-tasks.sqlite")
openTaskStore(t, db)
return db
}
t.Run("forward_migrated_schema", func(t *testing.T) {
// A database written by a newer release is the state a rollback has to
// recognise before it hands the file to an older binary.
db := newStore(t)
if err := execRaw(t, db, `PRAGMA user_version = 99`); err != nil {
t.Fatal(err)
}
code, _, stderr := runRecoveryCmd(t, "validate", "-db", db)
if code == 0 {
t.Fatal("validate accepted a forward-migrated database")
}
if !strings.Contains(stderr, "newer than supported version") {
t.Errorf("refusal must name the schema version; stderr = %q", stderr)
}
})
t.Run("corrupt_file", func(t *testing.T) {
// A real database header over pages that are not a database. The magic
// is right, so this fails on the content rather than on the format, which
// is the shape a half-written or truncated store has.
db := filepath.Join(t.TempDir(), "archie.db-tasks.sqlite")
corrupt := append([]byte("SQLite format 3\x00"), bytes.Repeat([]byte{0xff}, 4096)...)
if err := os.WriteFile(db, corrupt, 0o600); err != nil {
t.Fatal(err)
}
code, _, stderr := runRecoveryCmd(t, "validate", "-db", db)
if code == 0 {
t.Fatal("validate accepted a corrupt database")
}
if !strings.Contains(stderr, db) {
t.Errorf("refusal must name the file; stderr = %q", stderr)
}
})
t.Run("not_a_database", func(t *testing.T) {
// Pointing -db at the wrong file is the common operator mistake, and the
// configured db_path is a different file from the task database the
// State Store owns.
db := filepath.Join(t.TempDir(), "archie.db")
if err := os.WriteFile(db, []byte("db_path, not the task database"), 0o600); err != nil {
t.Fatal(err)
}
code, _, stderr := runRecoveryCmd(t, "validate", "-db", db)
if code == 0 {
t.Fatal("validate accepted a file that is not a database")
}
if !strings.Contains(stderr, db) {
t.Errorf("refusal must name the file; stderr = %q", stderr)
}
})
t.Run("stored_resource_that_will_not_validate", func(t *testing.T) {
// A stored value that both gates refuse: the write path stops it at
// Decode (workflow.ExecutionSettings.Validate rejects a negative limit),
// and boot stops on it too -- startWorkflowExecutionSettings decodes the
// same document and exits 1. It can only get here by bypassing the write
// path, which is exactly the state the operator has no path back from.
db := newStore(t)
seedResource(t, openTaskStore(t, db), controlplane.WorkflowExecutionSettingsKind,
`{"max_model_tool_steps":-1,"max_runtime_seconds":60,"max_consecutive_gate_failures":3}`, "hand-edited")
code, _, stderr := runRecoveryCmd(t, "validate", "-db", db)
if code == 0 {
t.Fatal("validate accepted a stored resource the write path would refuse")
}
if !strings.Contains(stderr, controlplane.WorkflowExecutionSettingsKind) {
t.Errorf("refusal must name the resource kind; stderr = %q", stderr)
}
})
t.Run("missing_database", func(t *testing.T) {
db := newStore(t)
if err := os.Remove(db); err != nil {
t.Fatal(err)
}
code, _, stderr := runRecoveryCmd(t, "validate", "-db", db)
if code == 0 {
t.Fatal("validate accepted a database that does not exist")
}
if !strings.Contains(stderr, db) {
t.Errorf("refusal must name the file; stderr = %q", stderr)
}
})
t.Run("store_without_the_control_plane_table", func(t *testing.T) {
// A store written before the control plane existed holds no stored
// settings. The serving process creates the table on its next start, so
// refusing this file would send an operator to restore a snapshot for a
// database that would have started perfectly well.
dir := t.TempDir()
configPath := writeMinimalConfig(t, dir)
db := filepath.Join(dir, "archie.db-tasks.sqlite")
openTaskStore(t, db)
if err := execRaw(t, db, `DROP TABLE resources`); err != nil {
t.Fatal(err)
}
code, stdout, stderr := runRecoveryCmd(t, "validate", "-db", db, "-config", configPath)
if code != 0 {
t.Fatalf("validate exited %d: %s", code, stderr)
}
if !strings.Contains(stdout, "0 stored resources validate") {
t.Errorf("validate must report that there was nothing stored to check; stdout = %q", stdout)
}
})
}
// rollback is the one operation that closes the documented gap: with the State
// Store stopped, it replays the value an earlier revision recorded through the
// ordinary replace, so no rollback RPC is needed and the rollback itself is
// audited as one more revision.
func TestRecoveryRollbackReplaysTheRevisionThroughReplace(t *testing.T) {
dir := t.TempDir()
db := filepath.Join(dir, "archie.db-tasks.sqlite")
st := openTaskStore(t, db)
seedResource(t, st, controlplane.ModelRoleAssignmentsKind, `{"implement":"anthropic/claude"}`, "first")
seedResource(t, st, controlplane.ModelRoleAssignmentsKind, `{"implement":"openai/gpt"}`, "second")
code, stdout, stderr := runRecoveryCmd(t, "rollback", "-db", db, "-kind", controlplane.ModelRoleAssignmentsKind)
if code != 0 {
t.Fatalf("rollback exited %d: %s", code, stderr)
}
if !strings.Contains(stdout, controlplane.ModelRoleAssignmentsKind) {
t.Errorf("rollback must name the resource it replayed; stdout = %q", stdout)
}
resource, err := st.Resource(t.Context(), controlplane.ModelRoleAssignmentsKind)
if err != nil {
t.Fatal(err)
}
if string(resource.Value) != `{"implement":"anthropic/claude"}` {
t.Errorf("resource value = %s, want the first revision's value", resource.Value)
}
if resource.Version != 3 {
t.Errorf("resource version = %d, want a new revision (3) rather than a rewrite of the old one", resource.Version)
}
history, err := st.ResourceHistory(t.Context(), controlplane.ModelRoleAssignmentsKind, 0)
if err != nil {
t.Fatal(err)
}
if len(history) != 3 {
t.Fatalf("history has %d revisions, want the rollback recorded as the third", len(history))
}
if history[0].Source != archied.OfflineRollbackSource {
t.Errorf("newest revision source = %q, want %q so the rollback is auditable", history[0].Source, archied.OfflineRollbackSource)
}
}
// The revision to restore may be named, which is how an operator reaches past
// the most recent change without replaying it first.
func TestRecoveryRollbackRestoresANamedRevision(t *testing.T) {
dir := t.TempDir()
db := filepath.Join(dir, "archie.db-tasks.sqlite")
st := openTaskStore(t, db)
seedResource(t, st, controlplane.ModelRoleAssignmentsKind, `{"implement":"anthropic/claude"}`, "first")
seedResource(t, st, controlplane.ModelRoleAssignmentsKind, `{"implement":"openai/gpt"}`, "second")
seedResource(t, st, controlplane.ModelRoleAssignmentsKind, `{"implement":"google/gemini"}`, "third")
code, _, stderr := runRecoveryCmd(t, "rollback", "-db", db, "-kind", controlplane.ModelRoleAssignmentsKind, "-revision", "1")
if code != 0 {
t.Fatalf("rollback exited %d: %s", code, stderr)
}
resource, err := st.Resource(t.Context(), controlplane.ModelRoleAssignmentsKind)
if err != nil {
t.Fatal(err)
}
if string(resource.Value) != `{"implement":"anthropic/claude"}` {
t.Errorf("resource value = %s, want the value recorded at revision 1", resource.Value)
}
}
func TestRecoveryRollbackRefusesWhatItCannotReplay(t *testing.T) {
newStore := func(t *testing.T, revisions int) string {
t.Helper()
db := filepath.Join(t.TempDir(), "archie.db-tasks.sqlite")
st := openTaskStore(t, db)
for i := range revisions {
value := `{"implement":"anthropic/claude"}`
if i > 0 {
value = `{"implement":"openai/gpt"}`
}
seedResource(t, st, controlplane.ModelRoleAssignmentsKind, value, "seed")
}
return db
}
t.Run("unknown_kind", func(t *testing.T) {
db := newStore(t, 2)
code, _, stderr := runRecoveryCmd(t, "rollback", "-db", db, "-kind", "not-a-resource")
if code == 0 {
t.Fatal("rollback of an unknown resource kind succeeded")
}
if !strings.Contains(stderr, "not-a-resource") {
t.Errorf("refusal must name the kind; stderr = %q", stderr)
}
})
t.Run("no_earlier_revision", func(t *testing.T) {
db := newStore(t, 1)
code, _, stderr := runRecoveryCmd(t, "rollback", "-db", db, "-kind", controlplane.ModelRoleAssignmentsKind)
if code == 0 {
t.Fatal("rollback with no earlier revision succeeded")
}
if !strings.Contains(stderr, "no earlier revision") {
t.Errorf("refusal must say there is nothing to roll back to; stderr = %q", stderr)
}
})
t.Run("current_revision", func(t *testing.T) {
db := newStore(t, 2)
code, _, stderr := runRecoveryCmd(t, "rollback", "-db", db, "-kind", controlplane.ModelRoleAssignmentsKind, "-revision", "2")
if code == 0 {
t.Fatal("rollback of the current revision succeeded")
}
if !strings.Contains(stderr, "is the current version") {
t.Errorf("refusal must say the revision is already the current value; stderr = %q", stderr)
}
})
t.Run("live_state_store", func(t *testing.T) {
db := newStore(t, 2)
holdStoreOwnership(t, db)
code, _, stderr := runRecoveryCmd(t, "rollback", "-db", db, "-kind", controlplane.ModelRoleAssignmentsKind)
if code == 0 {
t.Fatal("rollback against a live State Store succeeded")
}
if !strings.Contains(stderr, "owned by another process") {
t.Errorf("refusal must say the database is owned by another process; stderr = %q", stderr)
}
})
}
// A positional argument that is not a subcommand used to be ignored, leaving
// an operator who mistyped a command watching a server start instead.
func TestRecoveryRejectsAnUnknownSubcommand(t *testing.T) {
var stdout, stderr bytes.Buffer
code := runRecovery([]string{"backp", "-db", "x"}, &stdout, &stderr)
if code == 0 {
t.Fatal("an unknown subcommand exited 0")
}
if !strings.Contains(stderr.String(), "backp") {
t.Errorf("refusal must name the unknown subcommand; stderr = %q", stderr.String())
}
}
// writeConfigWithLogFile is writeMinimalConfig plus the daemon's durable log
// destination, which only the daemon may bring into existence.
func writeConfigWithLogFile(t *testing.T, dir string) (string, string) {
t.Helper()
configPath := writeMinimalConfig(t, dir)
logFile := filepath.Join(dir, "logs", "archied.log")
f, err := os.OpenFile(configPath, os.O_APPEND|os.O_WRONLY, 0o600)
if err != nil {
t.Fatalf("open config to append: %v", err)
}
defer func() {
if err := f.Close(); err != nil {
t.Fatalf("close config: %v", err)
}
}()
if _, err := fmt.Fprintf(f, "\n[log]\nfile = %q\n", logFile); err != nil {
t.Fatalf("append [log]: %v", err)
}
return configPath, logFile
}
// seedStoreResources seeds the store the way the State Store seeds it at boot,
// from the same config file the operator hands validate: the command's verdict
// is about the pair, and a store whose settings never matched the config is a
// state boot does not recognise either.
//
// The seeding builds its control-plane server from the shared provider set
// (internal/infrastructure/workflowsteps), which is what every composition root
// registers -- including openStateStoreControlPlane, the root helper the
// recovery path uses. This test cannot call that unexported helper, so it
// registers the same set itself rather than an empty manager: the definitions
// are decoded with the vocabulary the served path validates against, which is
// what makes the seeded values the ones validate is asked about.
func seedStoreResources(t *testing.T, st *store.Store, configPath string) {
t.Helper()
doc, err := configuration.New(nil).Resolve(configPath, "")
if err != nil {
t.Fatalf("resolve %s: %v", configPath, err)
}
steps, err := workflowsteps.NewManager()
if err != nil {
t.Fatalf("register workflow step vocabulary: %v", err)
}
server, err := controlplane.NewServer(st, steps)
if err != nil {
t.Fatalf("build control plane server: %v", err)
}
if _, err := server.ImportConfig(t.Context(), doc.Config); err != nil {
t.Fatalf("seed store resources: %v", err)
}
}
// storedResourceCount reads the count validate reports out of its summary line.
func storedResourceCount(t *testing.T, stdout string) int {
t.Helper()
before, found := strings.CutSuffix(stdout, " stored resources validate\n")
if !found {
t.Fatalf("no resource count in %q", stdout)
}
fields := strings.Fields(before)
count, err := strconv.Atoi(fields[len(fields)-1])
if err != nil {
t.Fatalf("resource count in %q: %v", stdout, err)
}
return count
}
// seedResource writes one revision of a control-plane resource through the
// store, the way the control plane's replace does. The request ID is derived
// from the label and the revision it produces: the store deduplicates a
// repeated request ID, so a fixed one would make the second write of a series
// a silent no-op.
func seedResource(t *testing.T, st *store.Store, kind, value, label string) {
t.Helper()
current, err := st.Resource(t.Context(), kind)
expected := int64(0)
if err == nil {
expected = current.Version
}
var document any
if err := json.Unmarshal([]byte(value), &document); err != nil {
t.Fatal(err)
}
encoded, err := json.Marshal(document)
if err != nil {
t.Fatal(err)
}
if _, err := st.PutResource(t.Context(), store.ResourceWrite{
Kind: kind, Value: encoded, Actor: label, Source: "test",
RequestID: fmt.Sprintf("%s-%d", label, expected+1), ExpectedVersion: expected,
}); err != nil {
t.Fatalf("seed resource %s: %v", kind, err)
}
}
// execRaw runs one statement against the database file the way an operator
// editing a broken store would, bypassing every check the store applies.
func execRaw(t *testing.T, path, statement string) error {
t.Helper()
db, err := sql.Open("sqlite", path)
if err != nil {
return err
}
defer func() { _ = db.Close() }()
_, err = db.ExecContext(t.Context(), statement)
return err
}
+41 -12
View File
@@ -120,21 +120,50 @@ file values would restore the second source of truth the control plane exists to
remove. The cost is that a bad stored value stops the daemon starting, and the
value can only be corrected through the daemon's own API.
### Recovery: not yet built
### Recovery: offline commands on the State Store
There is no escape hatch. Nothing skips the control plane at boot, and the only
writer for a resource is `ControlPlaneService.Command` on a running State Store.
An operator whose stored settings will not validate has no supported path back
short of editing `archie.db` by hand.
Nothing skips the control plane at boot, and the only writer for a resource is
`ControlPlaneService.Command` on a running State Store -- which is exactly what
the daemon fails closed without. `archie-state-store` therefore carries the
offline recovery commands (`archie-core-6zw0`). They work on the task database
file directly -- the configured `db_path` with `-tasks.sqlite` appended, not
`db_path` itself -- and need neither `archied` nor the Web UI running: `validate`
reads the configuration file the daemon boots with, which is what makes its
verdict the daemon's verdict.
`archie-core-6zw0` is the missing piece: offline commands that back up, restore,
validate and roll back the database without `archied` or the Web UI. Until it
lands, treat a settings change that requires a restart as a change that can
prevent one, and take a copy of `archie.db` first.
| Command | What it does | State Store |
| --- | --- | --- |
| `backup -db F -out F` | writes a transactionally consistent snapshot (`VACUUM INTO`), verified as a usable store before it is published | may be running |
| `restore -db F -from F` | replaces the database with a snapshot and drops the replaced file's `-wal`/`-shm` | must be stopped |
| `validate -db F [-config C]` | runs the checks that stop archied starting: the file is readable, uncorrupted and no newer than the binary; every stored resource decodes under the definition that owns it; and `configuration.Validate` accepts the settings the daemon would read -- the stored ones, or the seed the State Store writes for a kind the file does not hold yet -- layered onto `C`, the check the daemon reports as `validate database settings` | may be running |
| `rollback -db F -kind K [-revision N]` | replays an earlier revision of a stored resource through the ordinary replace, recording the rollback as a new revision | must be stopped |
Resource history is the in-band remedy and does exist: every write records its
prior value, so an edit made through the dashboard or a channel can be restored
by replaying an earlier revision while the State Store is up.
The two validation layers `validate` reports are not the same check, and it
reports both rather than merging them: the write path's own decode is stricter
about shape (it rejects unknown fields boot's decode ignores), while boot is
stricter about meaning (only boot checks `dispatch.trigger`, and a positive
`poll_interval`). Either one refusing is a store somebody has to look at.
No rollback RPC exists and none is needed: a revision carries its own value, so
restoring one is the same replace the dashboard and the channels already use.
The offline command is that operation with the store stopped -- the one state in
which the daemon's own API cannot serve it.
"Must be stopped" is checked rather than assumed. The serving process takes an
exclusive flock on `<db>.lock` for its whole life, and `restore` and `rollback`
refuse with `task database is owned by another process` while another process
holds it. The read-only commands -- `backup` and `validate` -- take no lock and
are meant to run against a serving store: `backup` because the update installer
runs inside the process it is replacing and cannot stop the State Store, and
`validate` because an operator checks a running deployment with it. That
`backup` exemption is only sound while backup never replaces the database, so
naming the database as its own `-out` is refused. `scripts/archie-update-install`
takes its pre-flight snapshot through this subcommand rather than through its
own `sqlite3` shell, so the installer and an operator run one implementation.
`rollback` is the surgical route back from a bad stored value; `restore` is the
blunt one, and is also the way back from a schema migration a release cannot
grow out of.
## Gateway restart: two constraints learned the hard way
+13
View File
@@ -83,6 +83,13 @@ type boot struct {
cfg config.Config
log *slog.Logger
// stderrLog keeps setupLogging on stderr, never touching cfg.Log.File. An
// offline command reads the bootstrap config but is not the daemon: creating
// the deployment's log, appending a line stamped component="daemon", or
// rotating a log the daemon is writing are side effects a diagnosis must not
// have, and it needs no durable copy of its own.
stderrLog bool
loader *configuration.Loader
doc *configuration.Document
currentProvenance atomic.Pointer[configuration.Provenance]
@@ -297,6 +304,12 @@ func (b *boot) loadConfig(ctx context.Context, cfgPath, overlayPath string) erro
// must not take the daemon down with it.
func (b *boot) setupLogging() error {
cfg := b.cfg
if b.stderrLog {
// The stderr logger newBootstrap installed is the whole destination.
// Nothing below this line is set up: the feed, the task-log registry and
// the rotating file all belong to a running daemon.
return nil
}
logFeed := logging.NewFeed(1000)
b.logFeed = logFeed
// Task logs live alongside the store rather than under cfg.Log.File's
+10 -1
View File
@@ -315,7 +315,7 @@ type runArgs struct {
}
func parseArgs() (runArgs, bool) {
defaultCfg := filepath.Join(configHome(), "archie", "config.toml")
defaultCfg := DefaultConfigPath()
var args runArgs
flag.StringVar(&args.cfgPath, "config", defaultCfg, "path to a TOML/YAML config file or configuration directory")
flag.StringVar(&args.overlayPath, "config-overlay", "", "path to a TOML/YAML overlay file or configuration directory applied on top of -config")
@@ -918,6 +918,15 @@ func configHome() string {
return filepath.Join(home, ".config")
}
// DefaultConfigPath is the configuration the daemon reads when -config is not
// given, and the one every command that asks a question about the daemon has to
// read to be answering about the same deployment: it is the daemon's rule, not
// a second one. It is exported for exactly that reason -- an offline command
// resolving the default any other way answers about a different config file
// (os.UserConfigDir, notably, rejects a relative XDG_CONFIG_HOME outright and
// leaves the caller with no path at all).
func DefaultConfigPath() string { return filepath.Join(configHome(), "archie", "config.toml") }
// updateReportPath is where the update watchdog leaves the phase-2 outcome
// of an update for this identity to relay on its next launch. The identity is
// hashed so multiple identities sharing one daemon (see
+19 -1
View File
@@ -215,7 +215,25 @@ func (b *boot) openStateStore(ctx context.Context) error {
log.Error("configure bindings cipher", "err", err)
return err
}
st, err := openProductionTaskStore(ctx, taskDBPath(cfg.DBPath), store.WithBindingCipher(bindingCipher))
// Claim the database before opening it. This process is the file's owner
// for as long as it lives, and the claim is what an offline recovery
// command checks before it rewrites the file: without it, "the State Store
// is stopped" is a rule the operator is trusted to have followed rather
// than a fact the command can verify. A second State Store on the same
// database fails here, which is the single-owner invariant stated instead
// of merely documented.
path := taskDBPath(cfg.DBPath)
ownership, err := store.AcquireOwnership(path)
if err != nil {
log.Error("claim state store ownership", "err", err)
return err
}
b.addCleanup(func() {
if err := ownership.Release(); err != nil {
log.Error("release state store ownership", "err", err)
}
})
st, err := openProductionTaskStore(ctx, path, store.WithBindingCipher(bindingCipher))
if err != nil {
log.Error("open state store", "err", err)
return err
@@ -0,0 +1,280 @@
// state_store_recovery.go composes the offline recovery subcommands of the
// standalone archie-state-store process. They exist because the control plane
// fails closed: a stored setting that will not validate stops archied
// starting, and the in-band remedy -- replaying an earlier revision while the
// State Store is up -- is unavailable in exactly the case that needs it
// (docs/architecture/safe-change-and-recovery.md).
//
// Every operation works on the task database file directly, with the State
// Store stopped, and the ones that write refuse when another process owns the
// file. Backup is the exception and deliberately so: the update installer
// cannot stop the process that runs it, so the snapshot that makes a failed
// update reversible is taken against a serving store through SQLite's own
// consistent copy.
//
// The control-plane server both validate and rollback read through is built by
// openStateStoreControlPlane -- the same root helper the served path uses -- so
// the offline checks resolve the same workflow step vocabulary as the process
// whose verdict they predict, and a constructor error is a refusal here rather
// than a server that would validate against a different vocabulary.
package archied
import (
"context"
"errors"
"fmt"
"time"
"github.com/samcharles93/archie-core/internal/config"
pb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1"
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
"github.com/samcharles93/archie-core/internal/store"
)
// The audit identity every offline rollback records. A rollback is one more
// revision of the resource, written through the ordinary replace, so the
// operator can see who went back and when -- and can go forward again the same
// way, with the State Store up or down.
const (
OfflineRollbackSource = "archie-state-store rollback"
OfflineRollbackActor = "operator:offline-recovery"
)
// Recovery operations, as named on the command line.
const (
RecoveryBackup = "backup"
RecoveryRestore = "restore"
RecoveryValidate = "validate"
RecoveryRollback = "rollback"
)
// StateStoreRecoveryOptions are the process inputs for one offline recovery
// operation.
type StateStoreRecoveryOptions struct {
// Operation is one of backup, restore, validate, rollback.
Operation string
// DB is the task database file the State Store owns: the configured
// db_path with "-tasks.sqlite" appended, not the configured path itself.
DB string
// Out is the snapshot backup writes.
Out string
// From is the snapshot restore reads.
From string
// Config and Overlay are the configuration the daemon would boot with.
// validate asks whether the stored settings are a state archied starts on,
// and that question is about the stored values *layered onto this
// configuration*, so the file config is part of the answer.
Config string
Overlay string
// Kind is the control-plane resource kind rollback replays.
Kind string
// Revision is the revision rollback replays. Zero means the newest
// revision older than the one the resource carries now.
Revision int64
}
// RunStateStoreRecovery performs one offline operation on the task database and
// returns the line the command reports on stdout.
func RunStateStoreRecovery(ctx context.Context, options StateStoreRecoveryOptions) (string, error) {
switch options.Operation {
case RecoveryBackup:
if options.DB == "" || options.Out == "" {
return "", errors.New("backup requires -db and -out")
}
if err := store.Backup(ctx, options.DB, options.Out); err != nil {
return "", err
}
return fmt.Sprintf("backed up %s to %s", options.DB, options.Out), nil
case RecoveryRestore:
if options.DB == "" || options.From == "" {
return "", errors.New("restore requires -db and -from")
}
if err := store.Restore(ctx, options.DB, options.From); err != nil {
return "", err
}
return fmt.Sprintf("restored %s from %s; start the State Store again", options.DB, options.From), nil
case RecoveryValidate:
if options.DB == "" {
return "", errors.New("validate requires -db")
}
return validateStore(ctx, options)
case RecoveryRollback:
if options.DB == "" || options.Kind == "" {
return "", errors.New("rollback requires -db and -kind")
}
return rollbackResource(ctx, options)
default:
return "", fmt.Errorf("unknown recovery command %q", options.Operation)
}
}
// validateStore answers the question an operator actually has: would archied
// start against this file. It runs the checks that stop it, in the order boot
// meets them.
//
// First the file itself, since a database that is corrupt, newer than this
// binary, or not a database is one the serving process refuses to open. Then
// the writer's own per-resource validation, which is what a hand-edited value
// violated. Then boot's gate: the file config with every stored resource
// layered onto it, and configuration.Validate -- the check whose failure the
// daemon reports as "validate database settings".
//
// The two validation layers disagree in both directions, and both are reported
// rather than merged. The write path is stricter about shape (it rejects
// unknown fields boot's decode ignores), while boot is stricter about meaning
// (only boot checks dispatch.trigger and a positive poll interval, both of
// which the write path's own validators let through). A store either check
// refuses is a store somebody has to look at.
func validateStore(ctx context.Context, options StateStoreRecoveryOptions) (string, error) {
version, err := store.ValidateFile(ctx, options.DB)
if err != nil {
return "", err
}
st, err := store.OpenReadOnly(ctx, options.DB)
if err != nil {
return "", err
}
defer func() { _ = st.Close() }()
server, err := openStateStoreControlPlane(st)
if err != nil {
return "", err
}
// Is there a resources table at all? A store written before the control plane
// existed has none, and the State Store creates it and seeds every kind from
// this same config on its next start -- so there is nothing stored to check
// with the writer's validator, and boot's gate is the config check alone.
// That is the same stance the layering takes for a kind the table is missing:
// what the daemon reads for a kind nothing was ever stored for is the seed.
checked := 0
stored, err := st.StoresResources(ctx)
if err != nil {
return "", err
}
if stored {
if checked, err = server.ValidateStored(ctx); err != nil {
return "", err
}
}
base, err := bootConfig(ctx, options)
if err != nil {
return "", err
}
document := base
if stored {
document, _, err = server.StoredRuntimeConfig(ctx, base)
if err != nil {
return "", err
}
}
if err := configuration.Validate(&document); err != nil {
// The daemon's own wording, so an operator can match this failure to the
// one that stopped archied starting.
return "", fmt.Errorf("validate database settings: %w", err)
}
return fmt.Sprintf("%s is a valid store at schema version %d; %d stored resources validate", options.DB, version, checked), nil
}
// bootConfig resolves the configuration the daemon would boot with. validate is
// the only recovery command that asks a question about the process rather than
// the file, and that question is about the stored values layered onto this
// document, so the document is part of the answer.
//
// It reads that config with the daemon's stderr logger: resolving it is a
// diagnosis, and a diagnosis that opened, appended to or rotated cfg.Log.File
// would edit the deployment it is inspecting.
func bootConfig(ctx context.Context, options StateStoreRecoveryOptions) (config.Config, error) {
b := newBootstrap()
b.stderrLog = true
defer b.cleanup()
if err := b.loadConfig(ctx, options.Config, options.Overlay); err != nil {
return config.Config{}, err
}
return b.cfg, nil
}
// rollbackResource is the one operation the Web UI cannot be asked to perform
// here: the daemon fails closed on the bad value, so the store that would serve
// the rollback is the store that will not start. It replays the revision's own
// value through the ordinary replace, so no rollback RPC exists or is needed,
// and the rollback is audited as one more revision.
func rollbackResource(ctx context.Context, options StateStoreRecoveryOptions) (string, error) {
// Before the lock and before the open: opening a store creates the file,
// and a rollback pointed at a mistyped path must not leave an empty
// database where the operator expected theirs.
if err := store.RequireDatabase(options.DB); err != nil {
return "", err
}
ownership, err := store.AcquireOwnership(options.DB)
if err != nil {
return "", err
}
defer func() { _ = ownership.Release() }()
st, err := store.Open(ctx, options.DB)
if err != nil {
return "", err
}
defer func() { _ = st.Close() }()
server, err := openStateStoreControlPlane(st)
if err != nil {
return "", err
}
if !server.Owns(options.Kind) {
return "", fmt.Errorf("unknown resource kind %q", options.Kind)
}
current, err := st.Resource(ctx, options.Kind)
if errors.Is(err, store.ErrResourceNotFound) {
return "", fmt.Errorf("%s has no stored value to roll back", options.Kind)
}
if err != nil {
return "", fmt.Errorf("read %s: %w", options.Kind, err)
}
revision, err := revisionToReplay(ctx, st, options.Kind, current.Version, options.Revision)
if err != nil {
return "", err
}
replaced, err := server.Command(ctx, &pb.CommandRequest{
Kind: options.Kind,
Command: "replace",
ValueJson: revision.Value,
ExpectedVersion: current.Version,
Actor: OfflineRollbackActor,
Source: OfflineRollbackSource,
RequestId: fmt.Sprintf("recovery-rollback-%s-%d", options.Kind, time.Now().UTC().UnixNano()),
})
if err != nil {
return "", fmt.Errorf("replay %s at revision %d: %w", options.Kind, revision.Version, err)
}
return fmt.Sprintf("rolled back %s from version %d to the value recorded at version %d; the store now holds version %d",
options.Kind, current.Version, revision.Version, replaced.GetResource().GetVersion()), nil
}
// revisionToReplay picks the value to put back: the revision the operator
// named, or the newest one older than the value the resource carries now.
func revisionToReplay(ctx context.Context, st *store.Store, kind string, current, requested int64) (store.Resource, error) {
history, err := st.ResourceHistory(ctx, kind, 0)
if err != nil {
return store.Resource{}, fmt.Errorf("read %s history: %w", kind, err)
}
if requested > 0 {
for _, revision := range history {
if revision.Version != requested {
continue
}
if revision.Version == current {
return store.Resource{}, fmt.Errorf("%s revision %d is the current version; there is nothing to roll back to", kind, requested)
}
return revision, nil
}
return store.Resource{}, fmt.Errorf("%s has no revision %d", kind, requested)
}
for _, revision := range history {
if revision.Version < current {
return revision, nil
}
}
return store.Resource{}, fmt.Errorf("%s has no earlier revision to roll back to", kind)
}
+72 -6
View File
@@ -72,13 +72,9 @@ func (s *Server) ImportConfig(ctx context.Context, cfg config.Config) (map[strin
if !errors.Is(err, store.ErrResourceNotFound) {
return nil, err
}
value, err := json.Marshal(definition.Seed(cfg))
value, err := definition.seededValue(cfg)
if err != nil {
return nil, fmt.Errorf("seed %s: %w", definition.Kind, err)
}
value, err = definition.Decode(value)
if err != nil {
return nil, fmt.Errorf("seed %s: %w", definition.Kind, err)
return nil, err
}
resource, err = s.store.PutResource(ctx, store.ResourceWrite{Kind: definition.Kind, Value: value, Actor: "system:migration", Source: "legacy-config", RequestID: "import:" + definition.Kind, ExpectedVersion: 0, At: time.Now().UTC()})
if err != nil {
@@ -89,6 +85,76 @@ func (s *Server) ImportConfig(ctx context.Context, cfg config.Config) (map[strin
return versions, nil
}
// seededValue is the value the State Store writes for a kind it does not hold
// yet: the definition's seed derived from cfg, run through the same Decode a
// write applies. ImportConfig stores it; the offline config layering reads it
// for the same reason, so "what the daemon sees for a kind that was never
// stored" is one value rather than two.
func (d Definition) seededValue(cfg config.Config) ([]byte, error) {
value, err := json.Marshal(d.Seed(cfg))
if err != nil {
return nil, fmt.Errorf("seed %s: %w", d.Kind, err)
}
value, err = d.Decode(value)
if err != nil {
return nil, fmt.Errorf("seed %s: %w", d.Kind, err)
}
return value, nil
}
// seededValues is what the State Store would write for every kind, keyed by
// kind.
func (s *Server) seededValues(cfg config.Config) (map[string][]byte, error) {
values := make(map[string][]byte, len(s.ordered))
for _, definition := range s.ordered {
value, err := definition.seededValue(cfg)
if err != nil {
return nil, err
}
values[definition.Kind] = value
}
return values, nil
}
// Owns reports whether kind is a resource the control plane owns, so a caller
// acting on a stored value can refuse a kind nobody defines before it reaches
// the store.
func (s *Server) Owns(kind string) bool {
_, ok := s.definitions[kind]
return ok
}
// ValidateStored decodes every stored resource with the definition that owns
// it, using the same Decode the write path applies, and reports how many
// resources it checked. Errors name the kind and the revision it was stored at,
// which is what an operator needs to roll the value back.
//
// It is the write path's own check, not boot's: boot refuses on
// configuration.Validate over the stored values layered onto the file config,
// and the two disagree in both directions (this one rejects unknown fields
// boot's decode ignores, while only boot checks dispatch.trigger and a positive
// poll interval). A caller answering "would the daemon start" therefore needs
// StoredRuntimeConfig and configuration.Validate as well -- see
// validateStore in internal/app/archied/state_store_recovery.go.
func (s *Server) ValidateStored(ctx context.Context) (int, error) {
checked := 0
var failures []error
for _, definition := range s.ordered {
resource, err := s.store.Resource(ctx, definition.Kind)
if errors.Is(err, store.ErrResourceNotFound) {
continue
}
if err != nil {
return checked, fmt.Errorf("read %s: %w", definition.Kind, err)
}
checked++
if _, err := definition.Decode(resource.Value); err != nil {
failures = append(failures, fmt.Errorf("%s at version %d: %w", definition.Kind, resource.Version, err))
}
}
return checked, errors.Join(failures...)
}
func validateAs[T any](input []byte, validate func(T) error) error {
var value T
decoder := json.NewDecoder(bytes.NewReader(input))
+90 -18
View File
@@ -3,13 +3,24 @@ package controlplane
import (
"context"
"encoding/json"
"errors"
"fmt"
"time"
"github.com/samcharles93/archie-core/internal/config"
pb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1"
"github.com/samcharles93/archie-core/internal/store"
)
// resourceReader is the one read the layering performs: a resource kind's value
// and the version it came from. The gRPC client and the in-process server both
// provide it, so "what the database covers in the running config" has exactly
// one implementation. The offline validate runs the same code as boot, which is
// the only way its verdict can be boot's verdict.
type resourceReader interface {
query(ctx context.Context, kind string, decode func([]byte) error) (int64, error)
}
// RuntimeConfig applies restart-scoped database resources over bootstrap
// configuration. Values intentionally absent from a control-plane projection,
// such as secret-bearing MCP headers and channel update commands, remain
@@ -19,9 +30,70 @@ import (
// layers them in can report which version it is running
// (docs/prds/control-plane-apply-status.md).
func (c *Client) RuntimeConfig(ctx context.Context, base config.Config) (config.Config, map[string]int64, error) {
return runtimeConfigFrom(ctx, c, base)
}
// RuntimeChatConfig layers the stored channel settings over the file document's
// chat section.
func (c *Client) RuntimeChatConfig(ctx context.Context, base config.ChatConfig) (config.ChatConfig, int64, error) {
return runtimeChatConfigFrom(ctx, c, base)
}
// StoredRuntimeConfig is RuntimeConfig's layering over the store this server
// owns, for a caller holding the database file rather than a connection to it.
// Boot layers the stored settings onto the file config and then runs
// configuration.Validate, so an offline check of that database has to make the
// same first move with the same implementation; called with the config the
// daemon would load, the result is the document boot decides on.
//
// A kind the store does not hold is read as the seed the State Store would
// write for it from base (see storeReader.query), because that store seeds
// every kind before it serves.
func (s *Server) StoredRuntimeConfig(ctx context.Context, base config.Config) (config.Config, map[string]int64, error) {
seeds, err := s.seededValues(base)
if err != nil {
return config.Config{}, nil, err
}
return runtimeConfigFrom(ctx, storeReader{resources: s.store, seeds: seeds}, base)
}
// storeReader reads the server's own store. It mirrors what the gRPC read
// answers -- a kind that is stored is the value and version it holds -- with
// one difference: a kind the store does not hold yet is the value the State
// Store seeds for it, since the daemon dials a store that has already run
// ImportConfig over the same config.
type storeReader struct {
resources ResourceStore
seeds map[string][]byte
}
func (r storeReader) query(ctx context.Context, kind string, decode func([]byte) error) (int64, error) {
resource, err := r.resources.Resource(ctx, kind)
if errors.Is(err, store.ErrResourceNotFound) {
seed, ok := r.seeds[kind]
if !ok {
return 0, fmt.Errorf("read %s: %w", kind, store.ErrResourceNotFound)
}
if err := decode(seed); err != nil {
return 0, fmt.Errorf("decode %s seed: %w", kind, err)
}
// No version: nothing has been stored for this kind, so nothing has
// been applied either.
return 0, nil
}
if err != nil {
return 0, fmt.Errorf("read %s: %w", kind, err)
}
if err := decode(resource.Value); err != nil {
return 0, fmt.Errorf("decode %s: %w", kind, err)
}
return resource.Version, nil
}
func runtimeConfigFrom(ctx context.Context, reader resourceReader, base config.Config) (config.Config, map[string]int64, error) {
out := base.Clone()
versions := map[string]int64{}
if err := c.queryInto(ctx, versions, ProviderSettingsKind, func(value []byte) error {
if err := layerResource(ctx, reader, versions, ProviderSettingsKind, func(value []byte) error {
var providers map[string]providerDocument
if err := json.Unmarshal(value, &providers); err != nil {
return err
@@ -34,18 +106,18 @@ func (c *Client) RuntimeConfig(ctx context.Context, base config.Config) (config.
}); err != nil {
return config.Config{}, nil, err
}
if err := c.queryJSONInto(ctx, versions, ModelRoleAssignmentsKind, &out.Models); err != nil {
if err := layerResourceJSON(ctx, reader, versions, ModelRoleAssignmentsKind, &out.Models); err != nil {
return config.Config{}, nil, err
}
if err := c.queryJSONInto(ctx, versions, RepositoryPoliciesKind, &out.Repos); err != nil {
if err := layerResourceJSON(ctx, reader, versions, RepositoryPoliciesKind, &out.Repos); err != nil {
return config.Config{}, nil, err
}
chat, chatVersion, err := c.RuntimeChatConfig(ctx, out.Chat)
chat, chatVersion, err := runtimeChatConfigFrom(ctx, reader, out.Chat)
if err != nil {
return config.Config{}, nil, err
}
out.Chat, versions[ChannelSettingsKind] = chat, chatVersion
if err := c.queryInto(ctx, versions, SchedulingPolicyKind, func(value []byte) error {
if err := layerResource(ctx, reader, versions, SchedulingPolicyKind, func(value []byte) error {
var policy schedulingPolicy
if err := json.Unmarshal(value, &policy); err != nil {
return err
@@ -59,11 +131,11 @@ func (c *Client) RuntimeConfig(ctx context.Context, base config.Config) (config.
}); err != nil {
return config.Config{}, nil, err
}
return c.runtimeToolConfig(ctx, versions, out)
return runtimeToolConfigFrom(ctx, reader, versions, out)
}
func (c *Client) runtimeToolConfig(ctx context.Context, versions map[string]int64, out config.Config) (config.Config, map[string]int64, error) {
if err := c.queryInto(ctx, versions, ToolSettingsKind, func(value []byte) error {
func runtimeToolConfigFrom(ctx context.Context, reader resourceReader, versions map[string]int64, out config.Config) (config.Config, map[string]int64, error) {
if err := layerResource(ctx, reader, versions, ToolSettingsKind, func(value []byte) error {
var settings toolSettings
if err := json.Unmarshal(value, &settings); err != nil {
return err
@@ -81,7 +153,7 @@ func (c *Client) runtimeToolConfig(ctx context.Context, versions map[string]int6
}); err != nil {
return config.Config{}, nil, err
}
if err := c.queryInto(ctx, versions, PluginSettingsKind, func(value []byte) error {
if err := layerResource(ctx, reader, versions, PluginSettingsKind, func(value []byte) error {
var settings pluginSettings
if err := json.Unmarshal(value, &settings); err != nil {
return err
@@ -91,15 +163,15 @@ func (c *Client) runtimeToolConfig(ctx context.Context, versions map[string]int6
}); err != nil {
return config.Config{}, nil, err
}
if err := c.queryJSONInto(ctx, versions, ContainerRuntimePoliciesKind, &out.Containers); err != nil {
if err := layerResourceJSON(ctx, reader, versions, ContainerRuntimePoliciesKind, &out.Containers); err != nil {
return config.Config{}, nil, err
}
return out, versions, nil
}
func (c *Client) RuntimeChatConfig(ctx context.Context, base config.ChatConfig) (config.ChatConfig, int64, error) {
func runtimeChatConfigFrom(ctx context.Context, reader resourceReader, base config.ChatConfig) (config.ChatConfig, int64, error) {
out := base
version, err := c.query(ctx, ChannelSettingsKind, func(value []byte) error {
version, err := reader.query(ctx, ChannelSettingsKind, func(value []byte) error {
var settings channelSettings
if err := json.Unmarshal(value, &settings); err != nil {
return err
@@ -117,10 +189,10 @@ func (c *Client) RuntimeChatConfig(ctx context.Context, base config.ChatConfig)
return out, version, err
}
// queryInto decodes a resource and records the version it came from, so
// RuntimeConfig ends up holding the version of every kind it layered in.
func (c *Client) queryInto(ctx context.Context, versions map[string]int64, kind string, decode func([]byte) error) error {
version, err := c.query(ctx, kind, decode)
// layerResource decodes a resource and records the version it came from, so the
// layering ends up holding the version of every kind it applied.
func layerResource(ctx context.Context, reader resourceReader, versions map[string]int64, kind string, decode func([]byte) error) error {
version, err := reader.query(ctx, kind, decode)
if err != nil {
return err
}
@@ -128,8 +200,8 @@ func (c *Client) queryInto(ctx context.Context, versions map[string]int64, kind
return nil
}
func (c *Client) queryJSONInto(ctx context.Context, versions map[string]int64, kind string, target any) error {
return c.queryInto(ctx, versions, kind, func(value []byte) error { return json.Unmarshal(value, target) })
func layerResourceJSON(ctx context.Context, reader resourceReader, versions map[string]int64, kind string, target any) error {
return layerResource(ctx, reader, versions, kind, func(value []byte) error { return json.Unmarshal(value, target) })
}
// query returns the version of the resource it decoded, so callers that layer
+16 -4
View File
@@ -382,6 +382,13 @@ esac
`)
writeFakeCommand(t, fakeDir, "go", `
printf '%s\n' "go $*" >> "$ARCHIE_TEST_CALLS"
# The pre-flight task-database backup runs through the release's own
# archie-state-store subcommand, so a release whose recovery path cannot run
# must refuse the update.
if [ "$1" = "run" ]; then
[ "${ARCHIE_TEST_GO_FAILS_RUN:-}" = "1" ] && exit 1
exit 0
fi
out=""
while [ "$#" -gt 0 ]; do
if [ "$1" = "-o" ]; then out="$2"; break; fi
@@ -745,6 +752,10 @@ func TestUpdateWatchdogCyclesEveryUnitInDependencyOrder(t *testing.T) {
// value up produces an empty backup, and a rollback then restores that empty
// file over the database and deletes its -wal/-shm. A rollback that destroys
// the task store is worse than no rollback.
//
// The snapshot is taken by the release's own recovery subcommand rather than
// by hand-rolled shell, so the installer and an operator run the same code and
// the two cannot drift apart.
func TestUpdateInstallBacksUpTheSiblingTaskDatabase(t *testing.T) {
work := t.TempDir()
configured := filepath.Join(work, "archie.db")
@@ -764,7 +775,8 @@ func TestUpdateInstallBacksUpTheSiblingTaskDatabase(t *testing.T) {
"ARCHIE_TASK_DB_BACKUP": backup,
}, "db_path = \""+configured+"\"\n\n[containers]\nimage = 'registry.example/archie-agent:stable'\n", false)
assertCallContains(t, calls, "sqlite3", realStore, ".backup")
assertCallContains(t, calls, "go run ./cmd/archie-state-store backup", "-db "+realStore, "-out "+backup)
assertCallAbsent(t, calls, "sqlite3")
assertCallContains(t, calls, "systemd-run", "--setenv=ARCHIE_TASK_DB_PATH="+realStore)
// The zero-byte placeholder is not the store, and must never be the thing
// handed to the watchdog as ARCHIE_TASK_DB_PATH.
@@ -793,9 +805,9 @@ func TestUpdateInstallIgnoresDbPathFromOtherSections(t *testing.T) {
"ARCHIE_TASK_DB_BACKUP": backup,
}, "db_path = \""+configured+"\"\n\n[indexing]\ndb_path = \""+indexStore+"\"\n\n[containers]\nimage = 'registry.example/archie-agent:stable'\n", false)
assertCallContains(t, calls, "sqlite3", realStore)
assertCallContains(t, calls, "go run ./cmd/archie-state-store backup", "-db "+realStore)
for _, call := range calls {
if strings.Contains(call, "sqlite3") && strings.Contains(call, indexStore) {
if strings.Contains(call, "go run ./cmd/archie-state-store backup") && strings.Contains(call, indexStore) {
t.Errorf("backed up the indexing database %s: %q", indexStore, call)
}
}
@@ -815,7 +827,7 @@ func TestUpdateInstallRefusesWhenTaskDatabaseCannotBeBackedUp(t *testing.T) {
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0",
"ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0",
"ARCHIE_TEST_SQLITE3_FAILS": "1",
"ARCHIE_TEST_GO_FAILS_RUN": "1",
}, "db_path = \""+configured+"\"\n\n[containers]\nimage = 'registry.example/archie-agent:stable'\n", true)
if !strings.Contains(output, "task database") {
+265
View File
@@ -0,0 +1,265 @@
package store
import (
"context"
"database/sql"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
)
// This file is the offline half of the store's contract with its owner: the
// operations the State Store process cannot perform on itself, because they
// read or replace the very file it has open. They are what
// docs/prds/runtime-control-plane.md ("Bootstrap, migration, and recovery")
// asks for -- back up, restore and validate the database without archied or
// the Web UI -- and every one of them works on the file directly.
//
// The read-only operations never take the ownership lock, because they are
// what an operator runs against a serving store. The one that replaces the
// file does, because a rewrite under a running server is the only way an
// offline command can destroy a task store.
// OpenReadOnly opens an existing store for reading only: no schema is created
// and no migration runs. That matters for recovery -- the snapshot an update
// takes has to stay exactly what the release that wrote it expects, and
// migrating a copy during verification would hand a rollback a schema its
// binary refuses.
func OpenReadOnly(ctx context.Context, path string) (*Store, error) {
if err := RequireDatabase(path); err != nil {
return nil, err
}
db, err := sql.Open("sqlite", readonlyDSN(path))
if err != nil {
return nil, err
}
db.SetMaxOpenConns(1)
// sql.Open is lazy: the first statement is where a file that is not a
// database, a missing WAL sidecar or an unreadable path surfaces.
var tables int
if err := db.QueryRowContext(ctx, `SELECT count(*) FROM sqlite_master`).Scan(&tables); err != nil {
return nil, errors.Join(fmt.Errorf("read task database %s: %w", path, err), db.Close())
}
return &Store{db: db}, nil
}
// readonlyDSN reads the store without the write-ahead log pragma the serving
// process sets: a reader has no business changing the database's journal mode,
// and the read-only flag makes an accidental write an error rather than a
// silent modification of a file someone else owns.
func readonlyDSN(path string) string {
separator := "?"
if strings.HasSuffix(path, "?") || strings.HasSuffix(path, "&") {
separator = ""
} else if strings.Contains(path, "?") {
separator = "&"
}
return path + separator + "mode=ro&_pragma=busy_timeout(5000)"
}
// StoresResources reports whether the store's file carries the control-plane
// resources table. A store written before the control plane existed has none
// and holds no stored settings to validate; the serving process creates the
// table on its next start, so an offline check must tell that apart from
// resources it cannot read.
func (s *Store) StoresResources(ctx context.Context) (bool, error) {
var tables int
if err := s.db.QueryRowContext(ctx, `SELECT count(*) FROM sqlite_master WHERE type='table' AND name='resources'`).Scan(&tables); err != nil {
return false, fmt.Errorf("inspect store schema: %w", err)
}
return tables > 0, nil
}
// RequireDatabase reports whether path is an existing store file. A writer
// that must not create the file it fails to find asks this first: a recovery
// command pointed at a mistyped path has to refuse, not leave a fresh empty
// database where an operator expected theirs.
func RequireDatabase(path string) error {
if _, err := os.Stat(path); err != nil {
return fmt.Errorf("task database %s: %w", path, err)
}
return nil
}
// ValidateFile checks an existing store the way the serving process meets it
// on startup and reports the schema version it carries: the file is a SQLite
// database this binary can read, it is not corrupt, and its schema is not
// newer than this binary supports.
func ValidateFile(ctx context.Context, path string) (int, error) {
st, err := OpenReadOnly(ctx, path)
if err != nil {
return 0, err
}
defer func() { _ = st.Close() }()
return st.check(ctx)
}
// check runs the file-level checks. integrity_check returns one row per
// problem, or the single row "ok"; reading the first row is therefore the
// verdict rather than a sample.
func (s *Store) check(ctx context.Context) (int, error) {
var result string
if err := s.db.QueryRowContext(ctx, `PRAGMA integrity_check`).Scan(&result); err != nil {
return 0, fmt.Errorf("integrity check: %w", err)
}
if result != "ok" {
return 0, fmt.Errorf("integrity check: %s", result)
}
var version int
if err := s.db.QueryRowContext(ctx, `PRAGMA user_version`).Scan(&version); err != nil {
return 0, fmt.Errorf("read schema version: %w", err)
}
if version > taskSchemaVersion {
return 0, fmt.Errorf("database schema version %d is newer than supported version %d", version, taskSchemaVersion)
}
return version, nil
}
// Backup writes a transactionally consistent snapshot of database to snapshot,
// using SQLite's own copy, so a store that is being written to cannot produce
// a torn file.
//
// It deliberately does not take the ownership lock. The update installer runs
// inside the process it is updating and cannot stop the State Store, so the
// snapshot that makes a failed update reversible is taken against a serving
// store; a backup that insisted on an exclusively owned file would break the
// one caller that exists. The snapshot is written beside the destination and
// renamed into place, so an interrupted backup never replaces a snapshot that
// was good with one that is not.
func Backup(ctx context.Context, database, snapshot string) error {
if err := RequireDatabase(database); err != nil {
return err
}
// The exemption from the ownership lock below is only sound while backup
// never replaces the database. With -out naming the database it would
// rename the snapshot over a file a serving State Store may still have
// open, orphaning the WAL of the file it just unlinked -- so it is refused
// exactly as Restore refuses it.
if same, err := sameFile(database, snapshot); err != nil {
return err
} else if same {
return fmt.Errorf("snapshot %s is the database itself", snapshot)
}
if err := os.MkdirAll(filepath.Dir(snapshot), 0o755); err != nil {
return fmt.Errorf("create snapshot directory: %w", err)
}
db, err := sql.Open("sqlite", sqliteDSN(database))
if err != nil {
return fmt.Errorf("open task database %s: %w", database, err)
}
defer func() { _ = db.Close() }()
staging := snapshot + ".tmp"
if err := os.Remove(staging); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("clear staging snapshot %s: %w", staging, err)
}
if _, err := db.ExecContext(ctx, `VACUUM INTO ?`, staging); err != nil {
return errors.Join(fmt.Errorf("snapshot %s: %w", database, err), os.Remove(staging))
}
// A snapshot that is not itself a usable store is worse than no snapshot:
// the update path treats its existence as proof that a rollback is
// possible.
if _, err := ValidateFile(ctx, staging); err != nil {
return errors.Join(fmt.Errorf("snapshot is not a usable store: %w", err), os.Remove(staging))
}
if err := os.Rename(staging, snapshot); err != nil {
return errors.Join(fmt.Errorf("publish snapshot %s: %w", snapshot, err), os.Remove(staging))
}
return nil
}
// Restore replaces database with the snapshot at snapshot. The snapshot is
// verified before anything is destroyed, so a file that is not a usable store
// cannot take the database down with it, and the database file itself is
// replaced by a rename, so a failure leaves either the old file or the new one
// rather than a mixture. The replaced file's -wal/-shm are removed first, which
// the comment at that step explains: between the two calls the database is
// merely incomplete, and re-running the command completes it.
//
// It holds the ownership lock for its duration: rewriting a database a running
// State Store still has open would leave that process writing to an unlinked
// inode while the store served a file nobody owns.
func Restore(ctx context.Context, database, snapshot string) error {
if _, err := os.Stat(filepath.Dir(database)); err != nil {
return fmt.Errorf("restore destination %s: %w", database, err)
}
if same, err := sameFile(database, snapshot); err != nil {
return err
} else if same {
return fmt.Errorf("restore snapshot %s is the database itself", snapshot)
}
ownership, err := AcquireOwnership(database)
if err != nil {
return err
}
defer func() { _ = ownership.Release() }()
if _, err := ValidateFile(ctx, snapshot); err != nil {
return fmt.Errorf("snapshot is not a usable store: %w", err)
}
staging := database + ".restore.tmp"
if err := copyFile(snapshot, staging); err != nil {
return errors.Join(err, os.Remove(staging))
} // The replaced database's write-ahead log belongs to the file that is
// going away. It goes first: between the two calls the database is merely
// incomplete, whereas a WAL left beside the file it does not describe is
// the state SQLite cannot recover from.
for _, sidecar := range []string{"-wal", "-shm"} {
if err := os.Remove(database + sidecar); err != nil && !os.IsNotExist(err) {
return errors.Join(fmt.Errorf("remove %s%s: %w", database, sidecar, err), os.Remove(staging))
}
}
if err := os.Rename(staging, database); err != nil {
return errors.Join(fmt.Errorf("publish restored database %s: %w", database, err), os.Remove(staging))
}
return nil
}
func sameFile(a, b string) (bool, error) {
first, err := os.Stat(a)
if err != nil {
if os.IsNotExist(err) {
return false, nil
}
return false, err
}
second, err := os.Stat(b)
if err != nil {
if os.IsNotExist(err) {
return false, nil
}
return false, err
}
return os.SameFile(first, second), nil
}
// copyFile writes source over a new file at path, flushed to disk before it is
// returned: the rename that publishes it must not reach the directory entries
// ahead of the bytes they point at. The snapshot's own permissions are kept, so
// a restore does not silently change who may read the operator's database.
func copyFile(source, path string) (retErr error) {
info, err := os.Stat(source)
if err != nil {
return fmt.Errorf("stat snapshot %s: %w", source, err)
}
in, err := os.Open(source)
if err != nil {
return fmt.Errorf("open snapshot %s: %w", source, err)
}
defer func() { retErr = errors.Join(retErr, in.Close()) }()
out, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, info.Mode().Perm())
if err != nil {
return fmt.Errorf("create %s: %w", path, err)
}
defer func() { retErr = errors.Join(retErr, out.Close()) }()
if _, err := io.Copy(out, in); err != nil {
return fmt.Errorf("write %s: %w", path, err)
}
if err := out.Sync(); err != nil {
return fmt.Errorf("sync %s: %w", path, err)
}
return nil
}
+92
View File
@@ -0,0 +1,92 @@
package store
import (
"context"
"database/sql"
"errors"
"fmt"
"os"
"path/filepath"
"testing"
)
// Ownership is what makes "the State Store is stopped" a fact an offline
// command can check, so the primitive itself is asserted here rather than only
// through the commands that use it: exclusivity, release, and per-database
// independence.
func TestOwnershipIsExclusivePerDatabase(t *testing.T) {
dir := t.TempDir()
first := filepath.Join(dir, "archie.db-tasks.sqlite")
second := filepath.Join(dir, "other.db-tasks.sqlite")
held, err := AcquireOwnership(first)
if err != nil {
t.Fatalf("first AcquireOwnership: %v", err)
}
if _, err := AcquireOwnership(first); !errors.Is(err, ErrStoreOwned) {
t.Fatalf("second AcquireOwnership on the same database = %v, want ErrStoreOwned", err)
}
// A different database is a different claim: one process owning its store
// must not stop another from owning its own.
elsewhere, err := AcquireOwnership(second)
if err != nil {
t.Fatalf("AcquireOwnership on an unrelated database: %v", err)
}
if _, err := os.Stat(first + ".lock"); err != nil {
t.Fatalf("ownership left no sidecar to lock: %v", err)
}
if err := held.Release(); err != nil {
t.Fatalf("Release: %v", err)
}
reacquired, err := AcquireOwnership(first)
if err != nil {
t.Fatalf("AcquireOwnership after Release: %v", err)
}
if err := errors.Join(reacquired.Release(), elsewhere.Release()); err != nil {
t.Fatalf("release on cleanup: %v", err)
}
}
// The snapshot an update takes is the state a rollback restores, and it is
// restored by the release that wrote it. Verifying the copy must therefore not
// migrate it: a snapshot silently upgraded to this binary's schema would be
// refused by the older binary it exists to serve.
func TestBackupSnapshotKeepsTheSchemaVersionItCopied(t *testing.T) {
dir := t.TempDir()
database := filepath.Join(dir, "archie.db-tasks.sqlite")
snapshot := filepath.Join(dir, "snapshot.sqlite")
st, err := Open(t.Context(), database)
if err != nil {
t.Fatal(err)
}
if err := st.Close(); err != nil {
t.Fatal(err)
}
// An older release's file: the schema version it wrote, which this binary
// is expected to migrate when the store is next opened for serving.
if err := setUserVersion(t.Context(), database, taskSchemaVersion-1); err != nil {
t.Fatal(err)
}
if err := Backup(t.Context(), database, snapshot); err != nil {
t.Fatalf("Backup: %v", err)
}
version, err := ValidateFile(t.Context(), snapshot)
if err != nil {
t.Fatalf("ValidateFile on the snapshot: %v", err)
}
if version != taskSchemaVersion-1 {
t.Fatalf("snapshot schema version = %d, want the %d it copied", version, taskSchemaVersion-1)
}
}
func setUserVersion(ctx context.Context, path string, version int) error {
db, err := sql.Open("sqlite", path)
if err != nil {
return err
}
defer func() { _ = db.Close() }()
_, err = db.ExecContext(ctx, fmt.Sprintf(`PRAGMA user_version = %d`, version))
return err
}
+68
View File
@@ -0,0 +1,68 @@
package store
import (
"errors"
"fmt"
"os"
"path/filepath"
)
// ErrStoreOwned is returned when a task database is already owned by another
// process. Ownership is what makes "the State Store is stopped" a fact an
// offline command can check rather than a rule the operator is trusted to
// follow: overwriting a database a running server still holds leaves it
// writing to an unlinked inode while the store serves a file nobody owns.
var ErrStoreOwned = errors.New("task database is owned by another process")
// Ownership is one process's exclusive claim over one task database, held for
// as long as that process owns the file. The serving State Store takes it for
// its whole life; the offline recovery commands take it for the duration of an
// operation that rewrites the file.
//
// The claim lives in a sidecar file beside the database rather than on the
// database inode itself, so it survives the rename a restore performs: after
// the swap the sidecar is still the one path every process locks.
type Ownership struct {
f *os.File
}
// AcquireOwnership takes the store's ownership lock without blocking. A
// database another process owns comes back as ErrStoreOwned, never as a wait:
// a process that must not proceed has to hear so, and a blocking lock would
// turn a mistyped command into a hung terminal.
func AcquireOwnership(path string) (*Ownership, error) {
lockPath := ownershipLockPath(path)
if err := os.MkdirAll(filepath.Dir(lockPath), 0o755); err != nil {
return nil, fmt.Errorf("create store directory for %s: %w", lockPath, err)
}
f, err := os.OpenFile(lockPath, os.O_RDWR|os.O_CREATE, 0o644)
if err != nil {
return nil, fmt.Errorf("open store lock %s: %w", lockPath, err)
}
taken, err := lockFile(f)
if err != nil {
return nil, errors.Join(fmt.Errorf("lock %s: %w", lockPath, err), f.Close())
}
if !taken {
return nil, errors.Join(fmt.Errorf("%s: %w", path, ErrStoreOwned), f.Close())
}
return &Ownership{f: f}, nil
}
// Release drops the claim and closes the sidecar. Closing the descriptor alone
// would release it too; the explicit unlock keeps the intent readable and
// makes Release idempotent.
func (o *Ownership) Release() error {
if o == nil || o.f == nil {
return nil
}
err := unlockFile(o.f)
f := o.f
o.f = nil
return errors.Join(err, f.Close())
}
// ownershipLockPath is the sidecar every owner of path locks. It is derived
// from the database path, so the State Store process and an offline command
// that were given the same database agree on it without configuration.
func ownershipLockPath(path string) string { return path + ".lock" }
+30
View File
@@ -0,0 +1,30 @@
//go:build unix
package store
import (
"errors"
"os"
"syscall"
)
// lockFile takes a non-blocking exclusive flock on f. The boolean reports
// whether it was taken: a lock another process holds (EWOULDBLOCK) is a normal
// answer to "is the State Store running?", not an error.
func lockFile(f *os.File) (bool, error) {
err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
if errors.Is(err, syscall.EWOULDBLOCK) {
return false, nil
}
if err != nil {
return false, err
}
return true, nil
}
func unlockFile(f *os.File) error {
if err := syscall.Flock(int(f.Fd()), syscall.LOCK_UN); err != nil {
return err
}
return nil
}
+23
View File
@@ -0,0 +1,23 @@
//go:build windows
// flock is a POSIX primitive and Windows is not a supported deployment surface
// (CLAUDE.md's deployment model is Linux and macOS), so this file fails closed:
// every lock attempt reports an error rather than silently degrading to "no
// lock". An offline command that cannot prove it owns the database must refuse
// to rewrite it, which is the only safe reading of an unavailable primitive.
//
// If a Windows deployment ever becomes supported, swap this for a real
// LockFileEx/UnlockFileEx pair; nothing else in the package needs to change.
package store
import (
"errors"
"os"
)
func lockFile(*os.File) (bool, error) {
return false, errors.New("store ownership locking requires a POSIX flock, which this platform does not provide")
}
func unlockFile(*os.File) error { return nil }
+9 -1
View File
@@ -183,10 +183,18 @@ fi
# that cannot take one must not start: once binaries are replaced, a failure
# leaves the previous release unable to read the store it just migrated. This
# runs ahead of the adapter refresh below so a refusal changes nothing at all.
#
# The snapshot is taken by the recovery subcommand of the release being
# installed, from its own checkout, rather than by hand-rolled sqlite3 shell:
# the operator running `archie-state-store backup` by hand and the installer
# are then the same implementation, and the two cannot drift. The installed
# binary is the previous release, which may predate the subcommand, so it
# cannot be the one asked -- and a release whose recovery path will not build is a
# release that cannot back up, which is exactly what this pre-flight refuses.
if [[ ",$components," == *,daemon,* ]] && [ -n "$TASK_DB_PATH" ] && [ -f "$TASK_DB_PATH" ]; then
echo "==> backing up task database for schema-safe rollback"
mkdir -p "$(dirname "$TASK_DB_BACKUP")"
if ! sqlite3 "$TASK_DB_PATH" ".backup '$TASK_DB_BACKUP'"; then
if ! go run ./cmd/archie-state-store backup -db "$TASK_DB_PATH" -out "$TASK_DB_BACKUP"; then
echo "refusing to install $GW: could not back up the task database at $TASK_DB_PATH, so a failed update could not be rolled back" >&2
exit 1
fi