feat(statestore): offline backup, restore, validate and rollback
Resolves archie-core-6zw0. Subcommands of cmd/archie-state-store (not a new binary, per organisation.md#process-boundaries) operating on the task database file directly with the State Store stopped: backup, restore, validate, rollback. A revision carries its value, so restoring replays it through the ordinary replace and no rollback RPC is added. The pre-flight backup in scripts/archie-update-install now calls the subcommand; the watchdog's refusal semantics are unchanged.
This commit is contained in:
commit
494580fad6
17 files changed
+1904
-43
No files matched your search
@@ -15,6 +15,7 @@ import (
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/app/archied"
|
||||
@@ -23,6 +24,13 @@ import (
|
||||
func main() { os.Exit(run()) }
|
||||
|
||||
func run() int {
|
||||
// A leading non-flag argument selects a recovery subcommand; anything else,
|
||||
// including no arguments at all, serves the State Store contract. An
|
||||
// unrecognised word is refused rather than ignored, so a mistyped recovery
|
||||
// command never leaves an operator watching a server start instead.
|
||||
if args := os.Args[1:]; len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||
return runRecovery(args, os.Stdout, os.Stderr)
|
||||
}
|
||||
base, err := os.UserConfigDir()
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
@@ -34,6 +42,10 @@ func run() int {
|
||||
flag.StringVar(&options.Listen, "listen", "", "state store gRPC listen address (defaults to [services.state].listen, else 127.0.0.1:9090)")
|
||||
flag.StringVar(&options.Token, "token", "", "bearer token required for a non-loopback listener (defaults to [services.state].target_token)")
|
||||
flag.StringVar(&options.ReadyAddr, "ready-addr", "", "optional readiness HTTP listen address (e.g. 127.0.0.1:9091)")
|
||||
flag.Usage = func() {
|
||||
fmt.Fprintf(flag.CommandLine.Output(), "usage: archie-state-store [flags] # serve the State Store gRPC contract\n\noffline recovery: archie-state-store <backup|restore|validate|rollback> -h\n\n")
|
||||
flag.PrintDefaults()
|
||||
}
|
||||
flag.Parse()
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/app/archied"
|
||||
)
|
||||
|
||||
// defaultConfigPath is the configuration the serve path reads by default, and
|
||||
// the one the daemon boots with. It is the daemon's own rule rather than a
|
||||
// second derivation of it: os.UserConfigDir (what the sibling processes use)
|
||||
// disagrees with archied on darwin and rejects a relative XDG_CONFIG_HOME
|
||||
// outright, which would leave validate answering about a file nobody boots.
|
||||
func defaultConfigPath() string { return archied.DefaultConfigPath() }
|
||||
|
||||
// recoveryUsage is the whole flag surface of the offline recovery commands.
|
||||
// They are subcommands of this binary rather than a recovery binary of their
|
||||
// own: offline database maintenance belongs to the process that owns the file
|
||||
// and its path layout (docs/architecture/organisation.md#process-boundaries).
|
||||
const recoveryUsage = `usage: archie-state-store <command> [flags]
|
||||
|
||||
With no command, archie-state-store serves the State Store gRPC contract.
|
||||
|
||||
The recovery commands operate on the task database file directly -- the
|
||||
configured db_path with "-tasks.sqlite" appended -- and are how an operator
|
||||
recovers a database the control plane will not start on. Stop the State Store
|
||||
first: restore and rollback rewrite the file and refuse while another process
|
||||
owns it, while backup takes a consistent snapshot of a serving store.
|
||||
|
||||
backup -db FILE -out FILE write a snapshot of the database
|
||||
restore -db FILE -from FILE replace the database with a snapshot
|
||||
validate -db FILE [-config FILE] check the file, its stored settings and the
|
||||
configuration the daemon would boot with
|
||||
rollback -db FILE -kind KIND replay an earlier revision of a stored
|
||||
[-revision N] resource through the ordinary replace
|
||||
`
|
||||
|
||||
// runRecovery parses and performs one offline recovery command, reporting the
|
||||
// operation's summary on stdout. Exit codes follow the serve path: 0 succeeded,
|
||||
// 1 the operation failed, 2 the command line was wrong.
|
||||
func runRecovery(args []string, stdout, stderr io.Writer) int {
|
||||
if len(args) == 0 {
|
||||
fmt.Fprint(stderr, recoveryUsage)
|
||||
return 2
|
||||
}
|
||||
command := args[0]
|
||||
options := archied.StateStoreRecoveryOptions{Operation: command}
|
||||
switch command {
|
||||
case archied.RecoveryBackup, archied.RecoveryRestore, archied.RecoveryValidate, archied.RecoveryRollback:
|
||||
default:
|
||||
fmt.Fprintf(stderr, "archie-state-store: unknown command %q\n\n", command)
|
||||
fmt.Fprint(stderr, recoveryUsage)
|
||||
return 2
|
||||
}
|
||||
|
||||
flags := flag.NewFlagSet("archie-state-store "+command, flag.ContinueOnError)
|
||||
flags.SetOutput(stderr)
|
||||
flags.Usage = func() { fmt.Fprint(stderr, recoveryUsage) }
|
||||
flags.StringVar(&options.DB, "db", "", "task database file the State Store owns (<db_path>-tasks.sqlite)")
|
||||
flags.StringVar(&options.Out, "out", "", "snapshot file backup writes")
|
||||
flags.StringVar(&options.From, "from", "", "snapshot file restore reads")
|
||||
flags.StringVar(&options.Kind, "kind", "", "control-plane resource kind rollback replays")
|
||||
flags.Int64Var(&options.Revision, "revision", 0, "revision rollback replays (default: the newest one older than the current value)")
|
||||
// Only validate asks a question about the process rather than the file, so
|
||||
// only validate takes the configuration the process would boot with.
|
||||
if command == archied.RecoveryValidate {
|
||||
flags.StringVar(&options.Config, "config", defaultConfigPath(), "configuration file or directory the daemon boots with")
|
||||
flags.StringVar(&options.Overlay, "config-overlay", "", "configuration overlay file or directory")
|
||||
}
|
||||
if err := flags.Parse(args[1:]); err != nil {
|
||||
// A requested help is not a wrong command line: the usage above is the
|
||||
// answer, and the serve path exits 0 for the same request.
|
||||
if errors.Is(err, flag.ErrHelp) {
|
||||
return 0
|
||||
}
|
||||
return 2
|
||||
}
|
||||
if flags.NArg() > 0 {
|
||||
fmt.Fprintf(stderr, "archie-state-store %s: unexpected argument %q\n", command, flags.Arg(0))
|
||||
return 2
|
||||
}
|
||||
|
||||
summary, err := archied.RunStateStoreRecovery(context.Background(), options)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "archie-state-store %s: %v\n", command, err)
|
||||
return 1
|
||||
}
|
||||
fmt.Fprintln(stdout, summary)
|
||||
return 0
|
||||
}
|
||||
@@ -0,0 +1,770 @@
|
||||
// Package main tests the offline recovery subcommands of archie-state-store:
|
||||
// backup, restore, validate and rollback against the task database file
|
||||
// directly. They are the operator's only path back when the control plane's
|
||||
// stored settings will not validate, because the daemon fails closed: the
|
||||
// in-band remedy (replay an earlier revision while the State Store is up) is
|
||||
// unavailable in exactly the case that needs it.
|
||||
//
|
||||
// The tests drive the subcommands the way an operator does -- through
|
||||
// runRecovery, against real files on disk -- rather than through the internal
|
||||
// helpers, so the flag surface and the exit codes are part of what is verified.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/app/archied"
|
||||
"github.com/samcharles93/archie-core/internal/app/controlplane"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/workflowsteps"
|
||||
"github.com/samcharles93/archie-core/internal/store"
|
||||
)
|
||||
|
||||
// runRecoveryCmd runs one subcommand the way main does and returns its exit
|
||||
// code with both streams captured.
|
||||
func runRecoveryCmd(t *testing.T, args ...string) (int, string, string) {
|
||||
t.Helper()
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := runRecovery(args, &stdout, &stderr)
|
||||
return code, stdout.String(), stderr.String()
|
||||
}
|
||||
|
||||
// openTaskStore opens the task database the subcommands operate on. It does not
|
||||
// take the ownership lock: only the serving process does that, and a test that
|
||||
// wants a "live" store holds it explicitly with holdStoreOwnership.
|
||||
func openTaskStore(t *testing.T, path string) *store.Store {
|
||||
t.Helper()
|
||||
st, err := store.Open(t.Context(), path)
|
||||
if err != nil {
|
||||
t.Fatalf("open task store %s: %v", path, err)
|
||||
}
|
||||
t.Cleanup(func() { _ = st.Close() })
|
||||
return st
|
||||
}
|
||||
|
||||
// holdStoreOwnership takes the ownership lock the running State Store holds for
|
||||
// its whole life, so a test can prove a command refuses to touch a live store.
|
||||
func holdStoreOwnership(t *testing.T, path string) {
|
||||
t.Helper()
|
||||
ownership, err := store.AcquireOwnership(path)
|
||||
if err != nil {
|
||||
t.Fatalf("acquire store ownership: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = ownership.Release() })
|
||||
}
|
||||
|
||||
// seedTask writes one task so a snapshot can be distinguished from the live
|
||||
// database by what it contains, not merely by its existence.
|
||||
func seedTask(t *testing.T, st *store.Store, title string) {
|
||||
t.Helper()
|
||||
if _, err := st.EnqueueChatTask(t.Context(), "acme", "widget", title, "body", "implement", ""); err != nil {
|
||||
t.Fatalf("seed task %q: %v", title, err)
|
||||
}
|
||||
}
|
||||
|
||||
func taskTitles(t *testing.T, path string) []string {
|
||||
t.Helper()
|
||||
st := openTaskStore(t, path)
|
||||
tasks, err := st.Tasks(t.Context(), 100)
|
||||
if err != nil {
|
||||
t.Fatalf("list tasks in %s: %v", path, err)
|
||||
}
|
||||
titles := make([]string, 0, len(tasks))
|
||||
for _, task := range tasks {
|
||||
titles = append(titles, task.Title)
|
||||
}
|
||||
return titles
|
||||
}
|
||||
|
||||
// backupPath is the snapshot every restore test restores from.
|
||||
func backupPath(t *testing.T, dir string) string { return filepath.Join(dir, "snapshot.sqlite") }
|
||||
|
||||
// backup takes the snapshot the caller then restores, through the subcommand
|
||||
// under test rather than through the store directly, so a broken backup fails
|
||||
// the restore test too instead of silently passing an empty file.
|
||||
func backup(t *testing.T, db, out string) {
|
||||
t.Helper()
|
||||
code, _, stderr := runRecoveryCmd(t, "backup", "-db", db, "-out", out)
|
||||
if code != 0 {
|
||||
t.Fatalf("backup exited %d: %s", code, stderr)
|
||||
}
|
||||
}
|
||||
|
||||
// The update installer cannot stop the process that runs it, so it backs the
|
||||
// task database up while the State Store is serving. VACUUM INTO is safe
|
||||
// against a live writer; a backup that insisted on an exclusively-owned file
|
||||
// would break the one caller it has, and the update would refuse to start.
|
||||
func TestRecoveryBackupSnapshotsAStoreItsStateStoreIsServing(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
db := filepath.Join(dir, "archie.db-tasks.sqlite")
|
||||
out := backupPath(t, dir)
|
||||
seedTask(t, openTaskStore(t, db), "before")
|
||||
holdStoreOwnership(t, db)
|
||||
if err := os.WriteFile(out, []byte("stale snapshot"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
code, stdout, stderr := runRecoveryCmd(t, "backup", "-db", db, "-out", out)
|
||||
if code != 0 {
|
||||
t.Fatalf("backup exited %d: %s", code, stderr)
|
||||
}
|
||||
if !strings.Contains(stdout, out) || !strings.Contains(stdout, db) {
|
||||
t.Errorf("backup must report the database and the snapshot it wrote; stdout = %q", stdout)
|
||||
}
|
||||
// The stale file was replaced by a real snapshot, and the snapshot is a
|
||||
// store a restore can put back: same rows, readable through the store.
|
||||
if got := taskTitles(t, out); len(got) != 1 || got[0] != "before" {
|
||||
t.Fatalf("snapshot tasks = %v, want the one seeded task", got)
|
||||
}
|
||||
if entries, err := filepath.Glob(out + ".tmp*"); err != nil || len(entries) != 0 {
|
||||
t.Errorf("backup left scratch files beside the snapshot: %v (%v)", entries, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The snapshot is the rollback state, so an interrupted backup must never
|
||||
// leave a truncated file where a good snapshot was.
|
||||
func TestRecoveryBackupRefusesAMissingDatabase(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
code, _, stderr := runRecoveryCmd(t, "backup", "-db", filepath.Join(dir, "absent.sqlite"), "-out", backupPath(t, dir))
|
||||
if code == 0 {
|
||||
t.Fatal("backup of a missing database succeeded")
|
||||
}
|
||||
if !strings.Contains(stderr, "absent.sqlite") {
|
||||
t.Errorf("refusal must name the database it could not read; stderr = %q", stderr)
|
||||
}
|
||||
}
|
||||
|
||||
// backup is the one recovery command allowed to run against a serving store,
|
||||
// and that exemption is only sound while it never replaces the database. With
|
||||
// -out naming the database it renames the snapshot over the live file and
|
||||
// leaves the WAL of the database it just unlinked -- the state the restore path
|
||||
// refuses up front and the code elsewhere calls unrecoverable.
|
||||
func TestRecoveryBackupRefusesTheDatabaseAsItsOwnSnapshot(t *testing.T) {
|
||||
db := filepath.Join(t.TempDir(), "archie.db-tasks.sqlite")
|
||||
st := openTaskStore(t, db)
|
||||
seedTask(t, st, "live")
|
||||
|
||||
code, _, stderr := runRecoveryCmd(t, "backup", "-db", db, "-out", db)
|
||||
if code == 0 {
|
||||
t.Fatal("backup over the database itself succeeded")
|
||||
}
|
||||
if !strings.Contains(stderr, "is the database itself") {
|
||||
t.Errorf("refusal must say the snapshot is the database; stderr = %q", stderr)
|
||||
}
|
||||
if got := taskTitles(t, db); len(got) != 1 || got[0] != "live" {
|
||||
t.Fatalf("refused backup changed the database: tasks = %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Restoring is the forward-migration escape hatch: the previous release
|
||||
// reads this file, so it must be exactly the snapshot, with the WAL that
|
||||
// belongs to the replaced database gone rather than replayed over it.
|
||||
func TestRecoveryRestorePutsTheSnapshotBackAndDropsItsWAL(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
db := filepath.Join(dir, "archie.db-tasks.sqlite")
|
||||
out := backupPath(t, dir)
|
||||
st := openTaskStore(t, db)
|
||||
seedTask(t, st, "before")
|
||||
backup(t, db, out)
|
||||
seedTask(t, st, "after")
|
||||
|
||||
code, stdout, stderr := runRecoveryCmd(t, "restore", "-db", db, "-from", out)
|
||||
if code != 0 {
|
||||
t.Fatalf("restore exited %d: %s", code, stderr)
|
||||
}
|
||||
if !strings.Contains(stdout, out) {
|
||||
t.Errorf("restore must report the snapshot it restored from; stdout = %q", stdout)
|
||||
}
|
||||
// Checked before anything reopens the database: opening a store sets WAL
|
||||
// mode, which creates these files, so the point is that the restore did not
|
||||
// leave the replaced database's log behind -- not that a later reader
|
||||
// cannot recreate its own.
|
||||
for _, suffix := range []string{"-wal", "-shm"} {
|
||||
if _, err := os.Stat(db + suffix); err == nil {
|
||||
t.Errorf("%s survived the restore: the replaced database's write-ahead log must not be replayed over the snapshot", db+suffix)
|
||||
}
|
||||
}
|
||||
if got := taskTitles(t, db); len(got) != 1 || got[0] != "before" {
|
||||
t.Fatalf("restored tasks = %v, want only the snapshot's task", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Overwriting a database a running State Store still owns leaves that process
|
||||
// writing to an unlinked inode while the store serves a file nobody owns --
|
||||
// the one way an offline command can destroy a task store.
|
||||
func TestRecoveryRestoreRefusesAStoreTheStateStoreIsRunning(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
db := filepath.Join(dir, "archie.db-tasks.sqlite")
|
||||
out := backupPath(t, dir)
|
||||
st := openTaskStore(t, db)
|
||||
seedTask(t, st, "before")
|
||||
backup(t, db, out)
|
||||
seedTask(t, st, "after")
|
||||
holdStoreOwnership(t, db)
|
||||
|
||||
code, _, stderr := runRecoveryCmd(t, "restore", "-db", db, "-from", out)
|
||||
if code == 0 {
|
||||
t.Fatal("restore against a live State Store succeeded")
|
||||
}
|
||||
if !strings.Contains(stderr, "owned by another process") {
|
||||
t.Errorf("refusal must say the database is owned by another process; stderr = %q", stderr)
|
||||
}
|
||||
if got := taskTitles(t, db); len(got) != 2 {
|
||||
t.Fatalf("refused restore changed the database: tasks = %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoveryRestoreRefusesAFileThatIsNotAStore(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
db := filepath.Join(dir, "archie.db-tasks.sqlite")
|
||||
seedTask(t, openTaskStore(t, db), "live")
|
||||
notAStore := filepath.Join(dir, "notes.txt")
|
||||
if err := os.WriteFile(notAStore, []byte("this is not a database"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
code, _, stderr := runRecoveryCmd(t, "restore", "-db", db, "-from", notAStore)
|
||||
if code == 0 {
|
||||
t.Fatal("restore from a file that is not a store succeeded")
|
||||
}
|
||||
if !strings.Contains(stderr, notAStore) {
|
||||
t.Errorf("refusal must name the snapshot it could not read; stderr = %q", stderr)
|
||||
}
|
||||
if got := taskTitles(t, db); len(got) != 1 {
|
||||
t.Fatalf("refused restore changed the database: tasks = %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoveryRestoreRefusesTheDatabaseAsItsOwnSnapshot(t *testing.T) {
|
||||
// Restoring a database from itself would delete the write-ahead log of the
|
||||
// file it is about to copy, so the one case that must never reach the swap
|
||||
// is refused up front.
|
||||
db := filepath.Join(t.TempDir(), "archie.db-tasks.sqlite")
|
||||
seedTask(t, openTaskStore(t, db), "live")
|
||||
|
||||
code, _, stderr := runRecoveryCmd(t, "restore", "-db", db, "-from", db)
|
||||
if code == 0 {
|
||||
t.Fatal("restore from the database itself succeeded")
|
||||
}
|
||||
if !strings.Contains(stderr, "is the database itself") {
|
||||
t.Errorf("refusal must say the snapshot is the database; stderr = %q", stderr)
|
||||
}
|
||||
if got := taskTitles(t, db); len(got) != 1 {
|
||||
t.Fatalf("refused restore changed the database: tasks = %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoveryValidateAcceptsAHealthyStore(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
configPath := writeMinimalConfig(t, dir)
|
||||
db := filepath.Join(dir, "archie.db-tasks.sqlite")
|
||||
st := openTaskStore(t, db)
|
||||
seedTask(t, st, "healthy")
|
||||
seedStoreResources(t, st, configPath)
|
||||
|
||||
code, stdout, stderr := runRecoveryCmd(t, "validate", "-db", db, "-config", configPath)
|
||||
if code != 0 {
|
||||
t.Fatalf("validate exited %d: %s", code, stderr)
|
||||
}
|
||||
if !strings.Contains(stdout, "schema version") {
|
||||
t.Errorf("validate must report the schema version it verified; stdout = %q", stdout)
|
||||
}
|
||||
// The count is part of what the operator reads, and a check that reports
|
||||
// "0 stored resources" for a store full of them is worse than no check.
|
||||
if count := storedResourceCount(t, stdout); count == 0 {
|
||||
t.Errorf("validate reported no stored resources for a seeded store; stdout = %q", stdout)
|
||||
}
|
||||
}
|
||||
|
||||
// validate's verdict is only useful if it is the daemon's verdict. Boot refuses
|
||||
// on configuration.Validate over the file config with every stored resource
|
||||
// layered onto it (internal/app/archied/control_plane.go), not on the write
|
||||
// path's own per-resource decode, and the two disagree: the write path accepts
|
||||
// a scheduling policy boot rejects.
|
||||
func TestRecoveryValidateRunsTheGateBootRuns(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
configPath := writeMinimalConfig(t, dir)
|
||||
db := filepath.Join(dir, "archie.db-tasks.sqlite")
|
||||
seedStoreResources(t, openTaskStore(t, db), configPath)
|
||||
|
||||
// The control: a store seeded from this config is one the daemon boots on.
|
||||
code, _, stderr := runRecoveryCmd(t, "validate", "-db", db, "-config", configPath)
|
||||
if code != 0 {
|
||||
t.Fatalf("validate exited %d on a store the daemon starts on: %s", code, stderr)
|
||||
}
|
||||
|
||||
// A hand-edited value, or an older revision of one: the write path's own
|
||||
// validator never inspects dispatch.trigger, so only the config gate
|
||||
// catches it, and the daemon exits 1 on it (validateDispatch).
|
||||
if err := execRaw(t, db, `UPDATE resources SET value='{"poll_interval":"1m","max_retries":3,"dispatch":{"trigger":"bogus"}}' WHERE kind='scheduling-policy'`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
code, stdout, stderr := runRecoveryCmd(t, "validate", "-db", db, "-config", configPath)
|
||||
if code == 0 {
|
||||
t.Fatalf("validate accepted a stored policy the daemon refuses to boot with; stdout = %q", stdout)
|
||||
}
|
||||
if !strings.Contains(stderr, "dispatch.trigger") {
|
||||
t.Errorf("refusal must name what boot rejects; stderr = %q", stderr)
|
||||
}
|
||||
|
||||
// A store the State Store never seeded holds no kinds at all. Its next start
|
||||
// creates the resources table and seeds every kind from this same config
|
||||
// (internal/app/archied/state_store.go), so archied layers those seeds and
|
||||
// starts: refusing the file here would send an operator to restore a
|
||||
// snapshot for a store the daemon boots on perfectly well.
|
||||
unseeded := filepath.Join(dir, "unseeded.db-tasks.sqlite")
|
||||
openTaskStore(t, unseeded)
|
||||
code, stdout, stderr = runRecoveryCmd(t, "validate", "-db", unseeded, "-config", configPath)
|
||||
if code != 0 {
|
||||
t.Fatalf("validate refused a store the State Store would seed on its next start; stderr = %q", stderr)
|
||||
}
|
||||
if !strings.Contains(stdout, "0 stored resources validate") {
|
||||
t.Errorf("validate must report that there was nothing stored to check; stdout = %q", stdout)
|
||||
}
|
||||
|
||||
// The upgrade case: a release defines a kind the store does not hold yet,
|
||||
// and validate runs before the State Store has started again. The State
|
||||
// Store seeds the kind it is missing, so this is the same stance as the
|
||||
// store with nothing stored at all -- the file is not the reason the daemon
|
||||
// would refuse to start, and saying otherwise costs the operator a snapshot.
|
||||
partial := filepath.Join(dir, "partial.db-tasks.sqlite")
|
||||
seedStoreResources(t, openTaskStore(t, partial), configPath)
|
||||
if err := execRaw(t, partial, `DELETE FROM resources WHERE kind='provider-settings'`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
code, stdout, stderr = runRecoveryCmd(t, "validate", "-db", partial, "-config", configPath)
|
||||
if code != 0 {
|
||||
t.Fatalf("validate refused a store missing a kind the State Store seeds; stderr = %q", stderr)
|
||||
}
|
||||
if count := storedResourceCount(t, stdout); count == 0 {
|
||||
t.Errorf("validate must still check the kinds the store does hold; stdout = %q", stdout)
|
||||
}
|
||||
}
|
||||
|
||||
// The subcommands advertise -h as the way to read their flags, and the serve
|
||||
// path in the same binary exits 0 for the same request, so a script probing the
|
||||
// recovery surface must not read a requested help as a failure.
|
||||
func TestRecoveryHelpExitsZero(t *testing.T) {
|
||||
commands := []string{archied.RecoveryBackup, archied.RecoveryRestore, archied.RecoveryValidate, archied.RecoveryRollback}
|
||||
for _, command := range commands {
|
||||
code, _, stderr := runRecoveryCmd(t, command, "-h")
|
||||
if code != 0 {
|
||||
t.Errorf("%s -h exited %d, want 0; stderr = %q", command, code, stderr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// validate diagnoses a deployment; it is not the deployment. Resolving the boot
|
||||
// config must not create, append to, or rotate the daemon's log file or the
|
||||
// directory it lives in: a line stamped component="daemon" in archied's log is
|
||||
// indistinguishable from the daemon having written it.
|
||||
func TestRecoveryValidateLeavesTheDaemonLogAlone(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
configPath, logFile := writeConfigWithLogFile(t, dir)
|
||||
db := filepath.Join(dir, "archie.db-tasks.sqlite")
|
||||
seedStoreResources(t, openTaskStore(t, db), configPath)
|
||||
|
||||
code, _, stderr := runRecoveryCmd(t, "validate", "-db", db, "-config", configPath)
|
||||
if code != 0 {
|
||||
t.Fatalf("validate exited %d: %s", code, stderr)
|
||||
}
|
||||
if _, err := os.Stat(logFile); !os.IsNotExist(err) {
|
||||
t.Fatalf("validate touched the daemon's log file %s (stat err = %v)", logFile, err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Dir(logFile)); !os.IsNotExist(err) {
|
||||
t.Errorf("validate created the log directory %s (stat err = %v)", filepath.Dir(logFile), err)
|
||||
}
|
||||
}
|
||||
|
||||
// validate's verdict is the daemon's verdict only if it reads the configuration
|
||||
// the daemon reads. archied resolves that default from its own configHome, which
|
||||
// honours a relative XDG_CONFIG_HOME; os.UserConfigDir rejects a relative one
|
||||
// outright, which left the command with no config and no verdict at all.
|
||||
func TestRecoveryDefaultConfigFollowsTheDaemonsConfigHome(t *testing.T) {
|
||||
relative := filepath.Join("relative", "config")
|
||||
t.Setenv("XDG_CONFIG_HOME", relative)
|
||||
if got, want := defaultConfigPath(), filepath.Join(relative, "archie", "config.toml"); got != want {
|
||||
t.Errorf("defaultConfigPath() with a relative XDG_CONFIG_HOME = %q, want %q", got, want)
|
||||
}
|
||||
|
||||
absolute := t.TempDir()
|
||||
t.Setenv("XDG_CONFIG_HOME", absolute)
|
||||
if got, want := defaultConfigPath(), filepath.Join(absolute, "archie", "config.toml"); got != want {
|
||||
t.Errorf("defaultConfigPath() = %q, want %q", got, want)
|
||||
}
|
||||
// The equality itself: one helper, so the two cannot drift apart again.
|
||||
if got, want := defaultConfigPath(), archied.DefaultConfigPath(); got != want {
|
||||
t.Errorf("defaultConfigPath() = %q, but the daemon resolves %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// validate exists to answer "would archied start against this file". Each case
|
||||
// below is a state the daemon fails closed on, or a file it cannot read at all.
|
||||
func TestRecoveryValidateRefusesWhatTheDaemonRefuses(t *testing.T) {
|
||||
newStore := func(t *testing.T) string {
|
||||
t.Helper()
|
||||
db := filepath.Join(t.TempDir(), "archie.db-tasks.sqlite")
|
||||
openTaskStore(t, db)
|
||||
return db
|
||||
}
|
||||
|
||||
t.Run("forward_migrated_schema", func(t *testing.T) {
|
||||
// A database written by a newer release is the state a rollback has to
|
||||
// recognise before it hands the file to an older binary.
|
||||
db := newStore(t)
|
||||
if err := execRaw(t, db, `PRAGMA user_version = 99`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
code, _, stderr := runRecoveryCmd(t, "validate", "-db", db)
|
||||
if code == 0 {
|
||||
t.Fatal("validate accepted a forward-migrated database")
|
||||
}
|
||||
if !strings.Contains(stderr, "newer than supported version") {
|
||||
t.Errorf("refusal must name the schema version; stderr = %q", stderr)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("corrupt_file", func(t *testing.T) {
|
||||
// A real database header over pages that are not a database. The magic
|
||||
// is right, so this fails on the content rather than on the format, which
|
||||
// is the shape a half-written or truncated store has.
|
||||
db := filepath.Join(t.TempDir(), "archie.db-tasks.sqlite")
|
||||
corrupt := append([]byte("SQLite format 3\x00"), bytes.Repeat([]byte{0xff}, 4096)...)
|
||||
if err := os.WriteFile(db, corrupt, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
code, _, stderr := runRecoveryCmd(t, "validate", "-db", db)
|
||||
if code == 0 {
|
||||
t.Fatal("validate accepted a corrupt database")
|
||||
}
|
||||
if !strings.Contains(stderr, db) {
|
||||
t.Errorf("refusal must name the file; stderr = %q", stderr)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("not_a_database", func(t *testing.T) {
|
||||
// Pointing -db at the wrong file is the common operator mistake, and the
|
||||
// configured db_path is a different file from the task database the
|
||||
// State Store owns.
|
||||
db := filepath.Join(t.TempDir(), "archie.db")
|
||||
if err := os.WriteFile(db, []byte("db_path, not the task database"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
code, _, stderr := runRecoveryCmd(t, "validate", "-db", db)
|
||||
if code == 0 {
|
||||
t.Fatal("validate accepted a file that is not a database")
|
||||
}
|
||||
if !strings.Contains(stderr, db) {
|
||||
t.Errorf("refusal must name the file; stderr = %q", stderr)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("stored_resource_that_will_not_validate", func(t *testing.T) {
|
||||
// A stored value that both gates refuse: the write path stops it at
|
||||
// Decode (workflow.ExecutionSettings.Validate rejects a negative limit),
|
||||
// and boot stops on it too -- startWorkflowExecutionSettings decodes the
|
||||
// same document and exits 1. It can only get here by bypassing the write
|
||||
// path, which is exactly the state the operator has no path back from.
|
||||
db := newStore(t)
|
||||
seedResource(t, openTaskStore(t, db), controlplane.WorkflowExecutionSettingsKind,
|
||||
`{"max_model_tool_steps":-1,"max_runtime_seconds":60,"max_consecutive_gate_failures":3}`, "hand-edited")
|
||||
code, _, stderr := runRecoveryCmd(t, "validate", "-db", db)
|
||||
if code == 0 {
|
||||
t.Fatal("validate accepted a stored resource the write path would refuse")
|
||||
}
|
||||
if !strings.Contains(stderr, controlplane.WorkflowExecutionSettingsKind) {
|
||||
t.Errorf("refusal must name the resource kind; stderr = %q", stderr)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("missing_database", func(t *testing.T) {
|
||||
db := newStore(t)
|
||||
if err := os.Remove(db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
code, _, stderr := runRecoveryCmd(t, "validate", "-db", db)
|
||||
if code == 0 {
|
||||
t.Fatal("validate accepted a database that does not exist")
|
||||
}
|
||||
if !strings.Contains(stderr, db) {
|
||||
t.Errorf("refusal must name the file; stderr = %q", stderr)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("store_without_the_control_plane_table", func(t *testing.T) {
|
||||
// A store written before the control plane existed holds no stored
|
||||
// settings. The serving process creates the table on its next start, so
|
||||
// refusing this file would send an operator to restore a snapshot for a
|
||||
// database that would have started perfectly well.
|
||||
dir := t.TempDir()
|
||||
configPath := writeMinimalConfig(t, dir)
|
||||
db := filepath.Join(dir, "archie.db-tasks.sqlite")
|
||||
openTaskStore(t, db)
|
||||
if err := execRaw(t, db, `DROP TABLE resources`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
code, stdout, stderr := runRecoveryCmd(t, "validate", "-db", db, "-config", configPath)
|
||||
if code != 0 {
|
||||
t.Fatalf("validate exited %d: %s", code, stderr)
|
||||
}
|
||||
if !strings.Contains(stdout, "0 stored resources validate") {
|
||||
t.Errorf("validate must report that there was nothing stored to check; stdout = %q", stdout)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// rollback is the one operation that closes the documented gap: with the State
|
||||
// Store stopped, it replays the value an earlier revision recorded through the
|
||||
// ordinary replace, so no rollback RPC is needed and the rollback itself is
|
||||
// audited as one more revision.
|
||||
func TestRecoveryRollbackReplaysTheRevisionThroughReplace(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
db := filepath.Join(dir, "archie.db-tasks.sqlite")
|
||||
st := openTaskStore(t, db)
|
||||
seedResource(t, st, controlplane.ModelRoleAssignmentsKind, `{"implement":"anthropic/claude"}`, "first")
|
||||
seedResource(t, st, controlplane.ModelRoleAssignmentsKind, `{"implement":"openai/gpt"}`, "second")
|
||||
|
||||
code, stdout, stderr := runRecoveryCmd(t, "rollback", "-db", db, "-kind", controlplane.ModelRoleAssignmentsKind)
|
||||
if code != 0 {
|
||||
t.Fatalf("rollback exited %d: %s", code, stderr)
|
||||
}
|
||||
if !strings.Contains(stdout, controlplane.ModelRoleAssignmentsKind) {
|
||||
t.Errorf("rollback must name the resource it replayed; stdout = %q", stdout)
|
||||
}
|
||||
resource, err := st.Resource(t.Context(), controlplane.ModelRoleAssignmentsKind)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(resource.Value) != `{"implement":"anthropic/claude"}` {
|
||||
t.Errorf("resource value = %s, want the first revision's value", resource.Value)
|
||||
}
|
||||
if resource.Version != 3 {
|
||||
t.Errorf("resource version = %d, want a new revision (3) rather than a rewrite of the old one", resource.Version)
|
||||
}
|
||||
history, err := st.ResourceHistory(t.Context(), controlplane.ModelRoleAssignmentsKind, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(history) != 3 {
|
||||
t.Fatalf("history has %d revisions, want the rollback recorded as the third", len(history))
|
||||
}
|
||||
if history[0].Source != archied.OfflineRollbackSource {
|
||||
t.Errorf("newest revision source = %q, want %q so the rollback is auditable", history[0].Source, archied.OfflineRollbackSource)
|
||||
}
|
||||
}
|
||||
|
||||
// The revision to restore may be named, which is how an operator reaches past
|
||||
// the most recent change without replaying it first.
|
||||
func TestRecoveryRollbackRestoresANamedRevision(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
db := filepath.Join(dir, "archie.db-tasks.sqlite")
|
||||
st := openTaskStore(t, db)
|
||||
seedResource(t, st, controlplane.ModelRoleAssignmentsKind, `{"implement":"anthropic/claude"}`, "first")
|
||||
seedResource(t, st, controlplane.ModelRoleAssignmentsKind, `{"implement":"openai/gpt"}`, "second")
|
||||
seedResource(t, st, controlplane.ModelRoleAssignmentsKind, `{"implement":"google/gemini"}`, "third")
|
||||
|
||||
code, _, stderr := runRecoveryCmd(t, "rollback", "-db", db, "-kind", controlplane.ModelRoleAssignmentsKind, "-revision", "1")
|
||||
if code != 0 {
|
||||
t.Fatalf("rollback exited %d: %s", code, stderr)
|
||||
}
|
||||
resource, err := st.Resource(t.Context(), controlplane.ModelRoleAssignmentsKind)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(resource.Value) != `{"implement":"anthropic/claude"}` {
|
||||
t.Errorf("resource value = %s, want the value recorded at revision 1", resource.Value)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoveryRollbackRefusesWhatItCannotReplay(t *testing.T) {
|
||||
newStore := func(t *testing.T, revisions int) string {
|
||||
t.Helper()
|
||||
db := filepath.Join(t.TempDir(), "archie.db-tasks.sqlite")
|
||||
st := openTaskStore(t, db)
|
||||
for i := range revisions {
|
||||
value := `{"implement":"anthropic/claude"}`
|
||||
if i > 0 {
|
||||
value = `{"implement":"openai/gpt"}`
|
||||
}
|
||||
seedResource(t, st, controlplane.ModelRoleAssignmentsKind, value, "seed")
|
||||
}
|
||||
return db
|
||||
}
|
||||
|
||||
t.Run("unknown_kind", func(t *testing.T) {
|
||||
db := newStore(t, 2)
|
||||
code, _, stderr := runRecoveryCmd(t, "rollback", "-db", db, "-kind", "not-a-resource")
|
||||
if code == 0 {
|
||||
t.Fatal("rollback of an unknown resource kind succeeded")
|
||||
}
|
||||
if !strings.Contains(stderr, "not-a-resource") {
|
||||
t.Errorf("refusal must name the kind; stderr = %q", stderr)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("no_earlier_revision", func(t *testing.T) {
|
||||
db := newStore(t, 1)
|
||||
code, _, stderr := runRecoveryCmd(t, "rollback", "-db", db, "-kind", controlplane.ModelRoleAssignmentsKind)
|
||||
if code == 0 {
|
||||
t.Fatal("rollback with no earlier revision succeeded")
|
||||
}
|
||||
if !strings.Contains(stderr, "no earlier revision") {
|
||||
t.Errorf("refusal must say there is nothing to roll back to; stderr = %q", stderr)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("current_revision", func(t *testing.T) {
|
||||
db := newStore(t, 2)
|
||||
code, _, stderr := runRecoveryCmd(t, "rollback", "-db", db, "-kind", controlplane.ModelRoleAssignmentsKind, "-revision", "2")
|
||||
if code == 0 {
|
||||
t.Fatal("rollback of the current revision succeeded")
|
||||
}
|
||||
if !strings.Contains(stderr, "is the current version") {
|
||||
t.Errorf("refusal must say the revision is already the current value; stderr = %q", stderr)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("live_state_store", func(t *testing.T) {
|
||||
db := newStore(t, 2)
|
||||
holdStoreOwnership(t, db)
|
||||
code, _, stderr := runRecoveryCmd(t, "rollback", "-db", db, "-kind", controlplane.ModelRoleAssignmentsKind)
|
||||
if code == 0 {
|
||||
t.Fatal("rollback against a live State Store succeeded")
|
||||
}
|
||||
if !strings.Contains(stderr, "owned by another process") {
|
||||
t.Errorf("refusal must say the database is owned by another process; stderr = %q", stderr)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// A positional argument that is not a subcommand used to be ignored, leaving
|
||||
// an operator who mistyped a command watching a server start instead.
|
||||
func TestRecoveryRejectsAnUnknownSubcommand(t *testing.T) {
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := runRecovery([]string{"backp", "-db", "x"}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("an unknown subcommand exited 0")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "backp") {
|
||||
t.Errorf("refusal must name the unknown subcommand; stderr = %q", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
// writeConfigWithLogFile is writeMinimalConfig plus the daemon's durable log
|
||||
// destination, which only the daemon may bring into existence.
|
||||
func writeConfigWithLogFile(t *testing.T, dir string) (string, string) {
|
||||
t.Helper()
|
||||
configPath := writeMinimalConfig(t, dir)
|
||||
logFile := filepath.Join(dir, "logs", "archied.log")
|
||||
f, err := os.OpenFile(configPath, os.O_APPEND|os.O_WRONLY, 0o600)
|
||||
if err != nil {
|
||||
t.Fatalf("open config to append: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := f.Close(); err != nil {
|
||||
t.Fatalf("close config: %v", err)
|
||||
}
|
||||
}()
|
||||
if _, err := fmt.Fprintf(f, "\n[log]\nfile = %q\n", logFile); err != nil {
|
||||
t.Fatalf("append [log]: %v", err)
|
||||
}
|
||||
return configPath, logFile
|
||||
}
|
||||
|
||||
// seedStoreResources seeds the store the way the State Store seeds it at boot,
|
||||
// from the same config file the operator hands validate: the command's verdict
|
||||
// is about the pair, and a store whose settings never matched the config is a
|
||||
// state boot does not recognise either.
|
||||
//
|
||||
// The seeding builds its control-plane server from the shared provider set
|
||||
// (internal/infrastructure/workflowsteps), which is what every composition root
|
||||
// registers -- including openStateStoreControlPlane, the root helper the
|
||||
// recovery path uses. This test cannot call that unexported helper, so it
|
||||
// registers the same set itself rather than an empty manager: the definitions
|
||||
// are decoded with the vocabulary the served path validates against, which is
|
||||
// what makes the seeded values the ones validate is asked about.
|
||||
func seedStoreResources(t *testing.T, st *store.Store, configPath string) {
|
||||
t.Helper()
|
||||
doc, err := configuration.New(nil).Resolve(configPath, "")
|
||||
if err != nil {
|
||||
t.Fatalf("resolve %s: %v", configPath, err)
|
||||
}
|
||||
steps, err := workflowsteps.NewManager()
|
||||
if err != nil {
|
||||
t.Fatalf("register workflow step vocabulary: %v", err)
|
||||
}
|
||||
server, err := controlplane.NewServer(st, steps)
|
||||
if err != nil {
|
||||
t.Fatalf("build control plane server: %v", err)
|
||||
}
|
||||
if _, err := server.ImportConfig(t.Context(), doc.Config); err != nil {
|
||||
t.Fatalf("seed store resources: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// storedResourceCount reads the count validate reports out of its summary line.
|
||||
func storedResourceCount(t *testing.T, stdout string) int {
|
||||
t.Helper()
|
||||
before, found := strings.CutSuffix(stdout, " stored resources validate\n")
|
||||
if !found {
|
||||
t.Fatalf("no resource count in %q", stdout)
|
||||
}
|
||||
fields := strings.Fields(before)
|
||||
count, err := strconv.Atoi(fields[len(fields)-1])
|
||||
if err != nil {
|
||||
t.Fatalf("resource count in %q: %v", stdout, err)
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
// seedResource writes one revision of a control-plane resource through the
|
||||
// store, the way the control plane's replace does. The request ID is derived
|
||||
// from the label and the revision it produces: the store deduplicates a
|
||||
// repeated request ID, so a fixed one would make the second write of a series
|
||||
// a silent no-op.
|
||||
func seedResource(t *testing.T, st *store.Store, kind, value, label string) {
|
||||
t.Helper()
|
||||
current, err := st.Resource(t.Context(), kind)
|
||||
expected := int64(0)
|
||||
if err == nil {
|
||||
expected = current.Version
|
||||
}
|
||||
var document any
|
||||
if err := json.Unmarshal([]byte(value), &document); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
encoded, err := json.Marshal(document)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := st.PutResource(t.Context(), store.ResourceWrite{
|
||||
Kind: kind, Value: encoded, Actor: label, Source: "test",
|
||||
RequestID: fmt.Sprintf("%s-%d", label, expected+1), ExpectedVersion: expected,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed resource %s: %v", kind, err)
|
||||
}
|
||||
}
|
||||
|
||||
// execRaw runs one statement against the database file the way an operator
|
||||
// editing a broken store would, bypassing every check the store applies.
|
||||
func execRaw(t *testing.T, path, statement string) error {
|
||||
t.Helper()
|
||||
db, err := sql.Open("sqlite", path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = db.Close() }()
|
||||
_, err = db.ExecContext(t.Context(), statement)
|
||||
return err
|
||||
}
|
||||
@@ -120,21 +120,50 @@ file values would restore the second source of truth the control plane exists to
|
||||
remove. The cost is that a bad stored value stops the daemon starting, and the
|
||||
value can only be corrected through the daemon's own API.
|
||||
|
||||
### Recovery: not yet built
|
||||
### Recovery: offline commands on the State Store
|
||||
|
||||
There is no escape hatch. Nothing skips the control plane at boot, and the only
|
||||
writer for a resource is `ControlPlaneService.Command` on a running State Store.
|
||||
An operator whose stored settings will not validate has no supported path back
|
||||
short of editing `archie.db` by hand.
|
||||
Nothing skips the control plane at boot, and the only writer for a resource is
|
||||
`ControlPlaneService.Command` on a running State Store -- which is exactly what
|
||||
the daemon fails closed without. `archie-state-store` therefore carries the
|
||||
offline recovery commands (`archie-core-6zw0`). They work on the task database
|
||||
file directly -- the configured `db_path` with `-tasks.sqlite` appended, not
|
||||
`db_path` itself -- and need neither `archied` nor the Web UI running: `validate`
|
||||
reads the configuration file the daemon boots with, which is what makes its
|
||||
verdict the daemon's verdict.
|
||||
|
||||
`archie-core-6zw0` is the missing piece: offline commands that back up, restore,
|
||||
validate and roll back the database without `archied` or the Web UI. Until it
|
||||
lands, treat a settings change that requires a restart as a change that can
|
||||
prevent one, and take a copy of `archie.db` first.
|
||||
| Command | What it does | State Store |
|
||||
| --- | --- | --- |
|
||||
| `backup -db F -out F` | writes a transactionally consistent snapshot (`VACUUM INTO`), verified as a usable store before it is published | may be running |
|
||||
| `restore -db F -from F` | replaces the database with a snapshot and drops the replaced file's `-wal`/`-shm` | must be stopped |
|
||||
| `validate -db F [-config C]` | runs the checks that stop archied starting: the file is readable, uncorrupted and no newer than the binary; every stored resource decodes under the definition that owns it; and `configuration.Validate` accepts the settings the daemon would read -- the stored ones, or the seed the State Store writes for a kind the file does not hold yet -- layered onto `C`, the check the daemon reports as `validate database settings` | may be running |
|
||||
| `rollback -db F -kind K [-revision N]` | replays an earlier revision of a stored resource through the ordinary replace, recording the rollback as a new revision | must be stopped |
|
||||
|
||||
Resource history is the in-band remedy and does exist: every write records its
|
||||
prior value, so an edit made through the dashboard or a channel can be restored
|
||||
by replaying an earlier revision while the State Store is up.
|
||||
The two validation layers `validate` reports are not the same check, and it
|
||||
reports both rather than merging them: the write path's own decode is stricter
|
||||
about shape (it rejects unknown fields boot's decode ignores), while boot is
|
||||
stricter about meaning (only boot checks `dispatch.trigger`, and a positive
|
||||
`poll_interval`). Either one refusing is a store somebody has to look at.
|
||||
|
||||
No rollback RPC exists and none is needed: a revision carries its own value, so
|
||||
restoring one is the same replace the dashboard and the channels already use.
|
||||
The offline command is that operation with the store stopped -- the one state in
|
||||
which the daemon's own API cannot serve it.
|
||||
|
||||
"Must be stopped" is checked rather than assumed. The serving process takes an
|
||||
exclusive flock on `<db>.lock` for its whole life, and `restore` and `rollback`
|
||||
refuse with `task database is owned by another process` while another process
|
||||
holds it. The read-only commands -- `backup` and `validate` -- take no lock and
|
||||
are meant to run against a serving store: `backup` because the update installer
|
||||
runs inside the process it is replacing and cannot stop the State Store, and
|
||||
`validate` because an operator checks a running deployment with it. That
|
||||
`backup` exemption is only sound while backup never replaces the database, so
|
||||
naming the database as its own `-out` is refused. `scripts/archie-update-install`
|
||||
takes its pre-flight snapshot through this subcommand rather than through its
|
||||
own `sqlite3` shell, so the installer and an operator run one implementation.
|
||||
|
||||
`rollback` is the surgical route back from a bad stored value; `restore` is the
|
||||
blunt one, and is also the way back from a schema migration a release cannot
|
||||
grow out of.
|
||||
|
||||
## Gateway restart: two constraints learned the hard way
|
||||
|
||||
|
||||
@@ -83,6 +83,13 @@ type boot struct {
|
||||
cfg config.Config
|
||||
log *slog.Logger
|
||||
|
||||
// stderrLog keeps setupLogging on stderr, never touching cfg.Log.File. An
|
||||
// offline command reads the bootstrap config but is not the daemon: creating
|
||||
// the deployment's log, appending a line stamped component="daemon", or
|
||||
// rotating a log the daemon is writing are side effects a diagnosis must not
|
||||
// have, and it needs no durable copy of its own.
|
||||
stderrLog bool
|
||||
|
||||
loader *configuration.Loader
|
||||
doc *configuration.Document
|
||||
currentProvenance atomic.Pointer[configuration.Provenance]
|
||||
@@ -297,6 +304,12 @@ func (b *boot) loadConfig(ctx context.Context, cfgPath, overlayPath string) erro
|
||||
// must not take the daemon down with it.
|
||||
func (b *boot) setupLogging() error {
|
||||
cfg := b.cfg
|
||||
if b.stderrLog {
|
||||
// The stderr logger newBootstrap installed is the whole destination.
|
||||
// Nothing below this line is set up: the feed, the task-log registry and
|
||||
// the rotating file all belong to a running daemon.
|
||||
return nil
|
||||
}
|
||||
logFeed := logging.NewFeed(1000)
|
||||
b.logFeed = logFeed
|
||||
// Task logs live alongside the store rather than under cfg.Log.File's
|
||||
|
||||
@@ -315,7 +315,7 @@ type runArgs struct {
|
||||
}
|
||||
|
||||
func parseArgs() (runArgs, bool) {
|
||||
defaultCfg := filepath.Join(configHome(), "archie", "config.toml")
|
||||
defaultCfg := DefaultConfigPath()
|
||||
var args runArgs
|
||||
flag.StringVar(&args.cfgPath, "config", defaultCfg, "path to a TOML/YAML config file or configuration directory")
|
||||
flag.StringVar(&args.overlayPath, "config-overlay", "", "path to a TOML/YAML overlay file or configuration directory applied on top of -config")
|
||||
@@ -918,6 +918,15 @@ func configHome() string {
|
||||
return filepath.Join(home, ".config")
|
||||
}
|
||||
|
||||
// DefaultConfigPath is the configuration the daemon reads when -config is not
|
||||
// given, and the one every command that asks a question about the daemon has to
|
||||
// read to be answering about the same deployment: it is the daemon's rule, not
|
||||
// a second one. It is exported for exactly that reason -- an offline command
|
||||
// resolving the default any other way answers about a different config file
|
||||
// (os.UserConfigDir, notably, rejects a relative XDG_CONFIG_HOME outright and
|
||||
// leaves the caller with no path at all).
|
||||
func DefaultConfigPath() string { return filepath.Join(configHome(), "archie", "config.toml") }
|
||||
|
||||
// updateReportPath is where the update watchdog leaves the phase-2 outcome
|
||||
// of an update for this identity to relay on its next launch. The identity is
|
||||
// hashed so multiple identities sharing one daemon (see
|
||||
|
||||
@@ -215,7 +215,25 @@ func (b *boot) openStateStore(ctx context.Context) error {
|
||||
log.Error("configure bindings cipher", "err", err)
|
||||
return err
|
||||
}
|
||||
st, err := openProductionTaskStore(ctx, taskDBPath(cfg.DBPath), store.WithBindingCipher(bindingCipher))
|
||||
// Claim the database before opening it. This process is the file's owner
|
||||
// for as long as it lives, and the claim is what an offline recovery
|
||||
// command checks before it rewrites the file: without it, "the State Store
|
||||
// is stopped" is a rule the operator is trusted to have followed rather
|
||||
// than a fact the command can verify. A second State Store on the same
|
||||
// database fails here, which is the single-owner invariant stated instead
|
||||
// of merely documented.
|
||||
path := taskDBPath(cfg.DBPath)
|
||||
ownership, err := store.AcquireOwnership(path)
|
||||
if err != nil {
|
||||
log.Error("claim state store ownership", "err", err)
|
||||
return err
|
||||
}
|
||||
b.addCleanup(func() {
|
||||
if err := ownership.Release(); err != nil {
|
||||
log.Error("release state store ownership", "err", err)
|
||||
}
|
||||
})
|
||||
st, err := openProductionTaskStore(ctx, path, store.WithBindingCipher(bindingCipher))
|
||||
if err != nil {
|
||||
log.Error("open state store", "err", err)
|
||||
return err
|
||||
|
||||
@@ -0,0 +1,280 @@
|
||||
// state_store_recovery.go composes the offline recovery subcommands of the
|
||||
// standalone archie-state-store process. They exist because the control plane
|
||||
// fails closed: a stored setting that will not validate stops archied
|
||||
// starting, and the in-band remedy -- replaying an earlier revision while the
|
||||
// State Store is up -- is unavailable in exactly the case that needs it
|
||||
// (docs/architecture/safe-change-and-recovery.md).
|
||||
//
|
||||
// Every operation works on the task database file directly, with the State
|
||||
// Store stopped, and the ones that write refuse when another process owns the
|
||||
// file. Backup is the exception and deliberately so: the update installer
|
||||
// cannot stop the process that runs it, so the snapshot that makes a failed
|
||||
// update reversible is taken against a serving store through SQLite's own
|
||||
// consistent copy.
|
||||
//
|
||||
// The control-plane server both validate and rollback read through is built by
|
||||
// openStateStoreControlPlane -- the same root helper the served path uses -- so
|
||||
// the offline checks resolve the same workflow step vocabulary as the process
|
||||
// whose verdict they predict, and a constructor error is a refusal here rather
|
||||
// than a server that would validate against a different vocabulary.
|
||||
package archied
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
pb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
|
||||
"github.com/samcharles93/archie-core/internal/store"
|
||||
)
|
||||
|
||||
// The audit identity every offline rollback records. A rollback is one more
|
||||
// revision of the resource, written through the ordinary replace, so the
|
||||
// operator can see who went back and when -- and can go forward again the same
|
||||
// way, with the State Store up or down.
|
||||
const (
|
||||
OfflineRollbackSource = "archie-state-store rollback"
|
||||
OfflineRollbackActor = "operator:offline-recovery"
|
||||
)
|
||||
|
||||
// Recovery operations, as named on the command line.
|
||||
const (
|
||||
RecoveryBackup = "backup"
|
||||
RecoveryRestore = "restore"
|
||||
RecoveryValidate = "validate"
|
||||
RecoveryRollback = "rollback"
|
||||
)
|
||||
|
||||
// StateStoreRecoveryOptions are the process inputs for one offline recovery
|
||||
// operation.
|
||||
type StateStoreRecoveryOptions struct {
|
||||
// Operation is one of backup, restore, validate, rollback.
|
||||
Operation string
|
||||
// DB is the task database file the State Store owns: the configured
|
||||
// db_path with "-tasks.sqlite" appended, not the configured path itself.
|
||||
DB string
|
||||
// Out is the snapshot backup writes.
|
||||
Out string
|
||||
// From is the snapshot restore reads.
|
||||
From string
|
||||
// Config and Overlay are the configuration the daemon would boot with.
|
||||
// validate asks whether the stored settings are a state archied starts on,
|
||||
// and that question is about the stored values *layered onto this
|
||||
// configuration*, so the file config is part of the answer.
|
||||
Config string
|
||||
Overlay string
|
||||
// Kind is the control-plane resource kind rollback replays.
|
||||
Kind string
|
||||
// Revision is the revision rollback replays. Zero means the newest
|
||||
// revision older than the one the resource carries now.
|
||||
Revision int64
|
||||
}
|
||||
|
||||
// RunStateStoreRecovery performs one offline operation on the task database and
|
||||
// returns the line the command reports on stdout.
|
||||
func RunStateStoreRecovery(ctx context.Context, options StateStoreRecoveryOptions) (string, error) {
|
||||
switch options.Operation {
|
||||
case RecoveryBackup:
|
||||
if options.DB == "" || options.Out == "" {
|
||||
return "", errors.New("backup requires -db and -out")
|
||||
}
|
||||
if err := store.Backup(ctx, options.DB, options.Out); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("backed up %s to %s", options.DB, options.Out), nil
|
||||
case RecoveryRestore:
|
||||
if options.DB == "" || options.From == "" {
|
||||
return "", errors.New("restore requires -db and -from")
|
||||
}
|
||||
if err := store.Restore(ctx, options.DB, options.From); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("restored %s from %s; start the State Store again", options.DB, options.From), nil
|
||||
case RecoveryValidate:
|
||||
if options.DB == "" {
|
||||
return "", errors.New("validate requires -db")
|
||||
}
|
||||
return validateStore(ctx, options)
|
||||
case RecoveryRollback:
|
||||
if options.DB == "" || options.Kind == "" {
|
||||
return "", errors.New("rollback requires -db and -kind")
|
||||
}
|
||||
return rollbackResource(ctx, options)
|
||||
default:
|
||||
return "", fmt.Errorf("unknown recovery command %q", options.Operation)
|
||||
}
|
||||
}
|
||||
|
||||
// validateStore answers the question an operator actually has: would archied
|
||||
// start against this file. It runs the checks that stop it, in the order boot
|
||||
// meets them.
|
||||
//
|
||||
// First the file itself, since a database that is corrupt, newer than this
|
||||
// binary, or not a database is one the serving process refuses to open. Then
|
||||
// the writer's own per-resource validation, which is what a hand-edited value
|
||||
// violated. Then boot's gate: the file config with every stored resource
|
||||
// layered onto it, and configuration.Validate -- the check whose failure the
|
||||
// daemon reports as "validate database settings".
|
||||
//
|
||||
// The two validation layers disagree in both directions, and both are reported
|
||||
// rather than merged. The write path is stricter about shape (it rejects
|
||||
// unknown fields boot's decode ignores), while boot is stricter about meaning
|
||||
// (only boot checks dispatch.trigger and a positive poll interval, both of
|
||||
// which the write path's own validators let through). A store either check
|
||||
// refuses is a store somebody has to look at.
|
||||
func validateStore(ctx context.Context, options StateStoreRecoveryOptions) (string, error) {
|
||||
version, err := store.ValidateFile(ctx, options.DB)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
st, err := store.OpenReadOnly(ctx, options.DB)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer func() { _ = st.Close() }()
|
||||
|
||||
server, err := openStateStoreControlPlane(st)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// Is there a resources table at all? A store written before the control plane
|
||||
// existed has none, and the State Store creates it and seeds every kind from
|
||||
// this same config on its next start -- so there is nothing stored to check
|
||||
// with the writer's validator, and boot's gate is the config check alone.
|
||||
// That is the same stance the layering takes for a kind the table is missing:
|
||||
// what the daemon reads for a kind nothing was ever stored for is the seed.
|
||||
checked := 0
|
||||
stored, err := st.StoresResources(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if stored {
|
||||
if checked, err = server.ValidateStored(ctx); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
|
||||
base, err := bootConfig(ctx, options)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
document := base
|
||||
if stored {
|
||||
document, _, err = server.StoredRuntimeConfig(ctx, base)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
if err := configuration.Validate(&document); err != nil {
|
||||
// The daemon's own wording, so an operator can match this failure to the
|
||||
// one that stopped archied starting.
|
||||
return "", fmt.Errorf("validate database settings: %w", err)
|
||||
}
|
||||
return fmt.Sprintf("%s is a valid store at schema version %d; %d stored resources validate", options.DB, version, checked), nil
|
||||
}
|
||||
|
||||
// bootConfig resolves the configuration the daemon would boot with. validate is
|
||||
// the only recovery command that asks a question about the process rather than
|
||||
// the file, and that question is about the stored values layered onto this
|
||||
// document, so the document is part of the answer.
|
||||
//
|
||||
// It reads that config with the daemon's stderr logger: resolving it is a
|
||||
// diagnosis, and a diagnosis that opened, appended to or rotated cfg.Log.File
|
||||
// would edit the deployment it is inspecting.
|
||||
func bootConfig(ctx context.Context, options StateStoreRecoveryOptions) (config.Config, error) {
|
||||
b := newBootstrap()
|
||||
b.stderrLog = true
|
||||
defer b.cleanup()
|
||||
if err := b.loadConfig(ctx, options.Config, options.Overlay); err != nil {
|
||||
return config.Config{}, err
|
||||
}
|
||||
return b.cfg, nil
|
||||
}
|
||||
|
||||
// rollbackResource is the one operation the Web UI cannot be asked to perform
|
||||
// here: the daemon fails closed on the bad value, so the store that would serve
|
||||
// the rollback is the store that will not start. It replays the revision's own
|
||||
// value through the ordinary replace, so no rollback RPC exists or is needed,
|
||||
// and the rollback is audited as one more revision.
|
||||
func rollbackResource(ctx context.Context, options StateStoreRecoveryOptions) (string, error) {
|
||||
// Before the lock and before the open: opening a store creates the file,
|
||||
// and a rollback pointed at a mistyped path must not leave an empty
|
||||
// database where the operator expected theirs.
|
||||
if err := store.RequireDatabase(options.DB); err != nil {
|
||||
return "", err
|
||||
}
|
||||
ownership, err := store.AcquireOwnership(options.DB)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer func() { _ = ownership.Release() }()
|
||||
|
||||
st, err := store.Open(ctx, options.DB)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer func() { _ = st.Close() }()
|
||||
|
||||
server, err := openStateStoreControlPlane(st)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !server.Owns(options.Kind) {
|
||||
return "", fmt.Errorf("unknown resource kind %q", options.Kind)
|
||||
}
|
||||
current, err := st.Resource(ctx, options.Kind)
|
||||
if errors.Is(err, store.ErrResourceNotFound) {
|
||||
return "", fmt.Errorf("%s has no stored value to roll back", options.Kind)
|
||||
}
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read %s: %w", options.Kind, err)
|
||||
}
|
||||
revision, err := revisionToReplay(ctx, st, options.Kind, current.Version, options.Revision)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
replaced, err := server.Command(ctx, &pb.CommandRequest{
|
||||
Kind: options.Kind,
|
||||
Command: "replace",
|
||||
ValueJson: revision.Value,
|
||||
ExpectedVersion: current.Version,
|
||||
Actor: OfflineRollbackActor,
|
||||
Source: OfflineRollbackSource,
|
||||
RequestId: fmt.Sprintf("recovery-rollback-%s-%d", options.Kind, time.Now().UTC().UnixNano()),
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("replay %s at revision %d: %w", options.Kind, revision.Version, err)
|
||||
}
|
||||
return fmt.Sprintf("rolled back %s from version %d to the value recorded at version %d; the store now holds version %d",
|
||||
options.Kind, current.Version, revision.Version, replaced.GetResource().GetVersion()), nil
|
||||
}
|
||||
|
||||
// revisionToReplay picks the value to put back: the revision the operator
|
||||
// named, or the newest one older than the value the resource carries now.
|
||||
func revisionToReplay(ctx context.Context, st *store.Store, kind string, current, requested int64) (store.Resource, error) {
|
||||
history, err := st.ResourceHistory(ctx, kind, 0)
|
||||
if err != nil {
|
||||
return store.Resource{}, fmt.Errorf("read %s history: %w", kind, err)
|
||||
}
|
||||
if requested > 0 {
|
||||
for _, revision := range history {
|
||||
if revision.Version != requested {
|
||||
continue
|
||||
}
|
||||
if revision.Version == current {
|
||||
return store.Resource{}, fmt.Errorf("%s revision %d is the current version; there is nothing to roll back to", kind, requested)
|
||||
}
|
||||
return revision, nil
|
||||
}
|
||||
return store.Resource{}, fmt.Errorf("%s has no revision %d", kind, requested)
|
||||
}
|
||||
for _, revision := range history {
|
||||
if revision.Version < current {
|
||||
return revision, nil
|
||||
}
|
||||
}
|
||||
return store.Resource{}, fmt.Errorf("%s has no earlier revision to roll back to", kind)
|
||||
}
|
||||
@@ -72,13 +72,9 @@ func (s *Server) ImportConfig(ctx context.Context, cfg config.Config) (map[strin
|
||||
if !errors.Is(err, store.ErrResourceNotFound) {
|
||||
return nil, err
|
||||
}
|
||||
value, err := json.Marshal(definition.Seed(cfg))
|
||||
value, err := definition.seededValue(cfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("seed %s: %w", definition.Kind, err)
|
||||
}
|
||||
value, err = definition.Decode(value)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("seed %s: %w", definition.Kind, err)
|
||||
return nil, err
|
||||
}
|
||||
resource, err = s.store.PutResource(ctx, store.ResourceWrite{Kind: definition.Kind, Value: value, Actor: "system:migration", Source: "legacy-config", RequestID: "import:" + definition.Kind, ExpectedVersion: 0, At: time.Now().UTC()})
|
||||
if err != nil {
|
||||
@@ -89,6 +85,76 @@ func (s *Server) ImportConfig(ctx context.Context, cfg config.Config) (map[strin
|
||||
return versions, nil
|
||||
}
|
||||
|
||||
// seededValue is the value the State Store writes for a kind it does not hold
|
||||
// yet: the definition's seed derived from cfg, run through the same Decode a
|
||||
// write applies. ImportConfig stores it; the offline config layering reads it
|
||||
// for the same reason, so "what the daemon sees for a kind that was never
|
||||
// stored" is one value rather than two.
|
||||
func (d Definition) seededValue(cfg config.Config) ([]byte, error) {
|
||||
value, err := json.Marshal(d.Seed(cfg))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("seed %s: %w", d.Kind, err)
|
||||
}
|
||||
value, err = d.Decode(value)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("seed %s: %w", d.Kind, err)
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
// seededValues is what the State Store would write for every kind, keyed by
|
||||
// kind.
|
||||
func (s *Server) seededValues(cfg config.Config) (map[string][]byte, error) {
|
||||
values := make(map[string][]byte, len(s.ordered))
|
||||
for _, definition := range s.ordered {
|
||||
value, err := definition.seededValue(cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
values[definition.Kind] = value
|
||||
}
|
||||
return values, nil
|
||||
}
|
||||
|
||||
// Owns reports whether kind is a resource the control plane owns, so a caller
|
||||
// acting on a stored value can refuse a kind nobody defines before it reaches
|
||||
// the store.
|
||||
func (s *Server) Owns(kind string) bool {
|
||||
_, ok := s.definitions[kind]
|
||||
return ok
|
||||
}
|
||||
|
||||
// ValidateStored decodes every stored resource with the definition that owns
|
||||
// it, using the same Decode the write path applies, and reports how many
|
||||
// resources it checked. Errors name the kind and the revision it was stored at,
|
||||
// which is what an operator needs to roll the value back.
|
||||
//
|
||||
// It is the write path's own check, not boot's: boot refuses on
|
||||
// configuration.Validate over the stored values layered onto the file config,
|
||||
// and the two disagree in both directions (this one rejects unknown fields
|
||||
// boot's decode ignores, while only boot checks dispatch.trigger and a positive
|
||||
// poll interval). A caller answering "would the daemon start" therefore needs
|
||||
// StoredRuntimeConfig and configuration.Validate as well -- see
|
||||
// validateStore in internal/app/archied/state_store_recovery.go.
|
||||
func (s *Server) ValidateStored(ctx context.Context) (int, error) {
|
||||
checked := 0
|
||||
var failures []error
|
||||
for _, definition := range s.ordered {
|
||||
resource, err := s.store.Resource(ctx, definition.Kind)
|
||||
if errors.Is(err, store.ErrResourceNotFound) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return checked, fmt.Errorf("read %s: %w", definition.Kind, err)
|
||||
}
|
||||
checked++
|
||||
if _, err := definition.Decode(resource.Value); err != nil {
|
||||
failures = append(failures, fmt.Errorf("%s at version %d: %w", definition.Kind, resource.Version, err))
|
||||
}
|
||||
}
|
||||
return checked, errors.Join(failures...)
|
||||
}
|
||||
|
||||
func validateAs[T any](input []byte, validate func(T) error) error {
|
||||
var value T
|
||||
decoder := json.NewDecoder(bytes.NewReader(input))
|
||||
|
||||
@@ -3,13 +3,24 @@ package controlplane
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
pb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1"
|
||||
"github.com/samcharles93/archie-core/internal/store"
|
||||
)
|
||||
|
||||
// resourceReader is the one read the layering performs: a resource kind's value
|
||||
// and the version it came from. The gRPC client and the in-process server both
|
||||
// provide it, so "what the database covers in the running config" has exactly
|
||||
// one implementation. The offline validate runs the same code as boot, which is
|
||||
// the only way its verdict can be boot's verdict.
|
||||
type resourceReader interface {
|
||||
query(ctx context.Context, kind string, decode func([]byte) error) (int64, error)
|
||||
}
|
||||
|
||||
// RuntimeConfig applies restart-scoped database resources over bootstrap
|
||||
// configuration. Values intentionally absent from a control-plane projection,
|
||||
// such as secret-bearing MCP headers and channel update commands, remain
|
||||
@@ -19,9 +30,70 @@ import (
|
||||
// layers them in can report which version it is running
|
||||
// (docs/prds/control-plane-apply-status.md).
|
||||
func (c *Client) RuntimeConfig(ctx context.Context, base config.Config) (config.Config, map[string]int64, error) {
|
||||
return runtimeConfigFrom(ctx, c, base)
|
||||
}
|
||||
|
||||
// RuntimeChatConfig layers the stored channel settings over the file document's
|
||||
// chat section.
|
||||
func (c *Client) RuntimeChatConfig(ctx context.Context, base config.ChatConfig) (config.ChatConfig, int64, error) {
|
||||
return runtimeChatConfigFrom(ctx, c, base)
|
||||
}
|
||||
|
||||
// StoredRuntimeConfig is RuntimeConfig's layering over the store this server
|
||||
// owns, for a caller holding the database file rather than a connection to it.
|
||||
// Boot layers the stored settings onto the file config and then runs
|
||||
// configuration.Validate, so an offline check of that database has to make the
|
||||
// same first move with the same implementation; called with the config the
|
||||
// daemon would load, the result is the document boot decides on.
|
||||
//
|
||||
// A kind the store does not hold is read as the seed the State Store would
|
||||
// write for it from base (see storeReader.query), because that store seeds
|
||||
// every kind before it serves.
|
||||
func (s *Server) StoredRuntimeConfig(ctx context.Context, base config.Config) (config.Config, map[string]int64, error) {
|
||||
seeds, err := s.seededValues(base)
|
||||
if err != nil {
|
||||
return config.Config{}, nil, err
|
||||
}
|
||||
return runtimeConfigFrom(ctx, storeReader{resources: s.store, seeds: seeds}, base)
|
||||
}
|
||||
|
||||
// storeReader reads the server's own store. It mirrors what the gRPC read
|
||||
// answers -- a kind that is stored is the value and version it holds -- with
|
||||
// one difference: a kind the store does not hold yet is the value the State
|
||||
// Store seeds for it, since the daemon dials a store that has already run
|
||||
// ImportConfig over the same config.
|
||||
type storeReader struct {
|
||||
resources ResourceStore
|
||||
seeds map[string][]byte
|
||||
}
|
||||
|
||||
func (r storeReader) query(ctx context.Context, kind string, decode func([]byte) error) (int64, error) {
|
||||
resource, err := r.resources.Resource(ctx, kind)
|
||||
if errors.Is(err, store.ErrResourceNotFound) {
|
||||
seed, ok := r.seeds[kind]
|
||||
if !ok {
|
||||
return 0, fmt.Errorf("read %s: %w", kind, store.ErrResourceNotFound)
|
||||
}
|
||||
if err := decode(seed); err != nil {
|
||||
return 0, fmt.Errorf("decode %s seed: %w", kind, err)
|
||||
}
|
||||
// No version: nothing has been stored for this kind, so nothing has
|
||||
// been applied either.
|
||||
return 0, nil
|
||||
}
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("read %s: %w", kind, err)
|
||||
}
|
||||
if err := decode(resource.Value); err != nil {
|
||||
return 0, fmt.Errorf("decode %s: %w", kind, err)
|
||||
}
|
||||
return resource.Version, nil
|
||||
}
|
||||
|
||||
func runtimeConfigFrom(ctx context.Context, reader resourceReader, base config.Config) (config.Config, map[string]int64, error) {
|
||||
out := base.Clone()
|
||||
versions := map[string]int64{}
|
||||
if err := c.queryInto(ctx, versions, ProviderSettingsKind, func(value []byte) error {
|
||||
if err := layerResource(ctx, reader, versions, ProviderSettingsKind, func(value []byte) error {
|
||||
var providers map[string]providerDocument
|
||||
if err := json.Unmarshal(value, &providers); err != nil {
|
||||
return err
|
||||
@@ -34,18 +106,18 @@ func (c *Client) RuntimeConfig(ctx context.Context, base config.Config) (config.
|
||||
}); err != nil {
|
||||
return config.Config{}, nil, err
|
||||
}
|
||||
if err := c.queryJSONInto(ctx, versions, ModelRoleAssignmentsKind, &out.Models); err != nil {
|
||||
if err := layerResourceJSON(ctx, reader, versions, ModelRoleAssignmentsKind, &out.Models); err != nil {
|
||||
return config.Config{}, nil, err
|
||||
}
|
||||
if err := c.queryJSONInto(ctx, versions, RepositoryPoliciesKind, &out.Repos); err != nil {
|
||||
if err := layerResourceJSON(ctx, reader, versions, RepositoryPoliciesKind, &out.Repos); err != nil {
|
||||
return config.Config{}, nil, err
|
||||
}
|
||||
chat, chatVersion, err := c.RuntimeChatConfig(ctx, out.Chat)
|
||||
chat, chatVersion, err := runtimeChatConfigFrom(ctx, reader, out.Chat)
|
||||
if err != nil {
|
||||
return config.Config{}, nil, err
|
||||
}
|
||||
out.Chat, versions[ChannelSettingsKind] = chat, chatVersion
|
||||
if err := c.queryInto(ctx, versions, SchedulingPolicyKind, func(value []byte) error {
|
||||
if err := layerResource(ctx, reader, versions, SchedulingPolicyKind, func(value []byte) error {
|
||||
var policy schedulingPolicy
|
||||
if err := json.Unmarshal(value, &policy); err != nil {
|
||||
return err
|
||||
@@ -59,11 +131,11 @@ func (c *Client) RuntimeConfig(ctx context.Context, base config.Config) (config.
|
||||
}); err != nil {
|
||||
return config.Config{}, nil, err
|
||||
}
|
||||
return c.runtimeToolConfig(ctx, versions, out)
|
||||
return runtimeToolConfigFrom(ctx, reader, versions, out)
|
||||
}
|
||||
|
||||
func (c *Client) runtimeToolConfig(ctx context.Context, versions map[string]int64, out config.Config) (config.Config, map[string]int64, error) {
|
||||
if err := c.queryInto(ctx, versions, ToolSettingsKind, func(value []byte) error {
|
||||
func runtimeToolConfigFrom(ctx context.Context, reader resourceReader, versions map[string]int64, out config.Config) (config.Config, map[string]int64, error) {
|
||||
if err := layerResource(ctx, reader, versions, ToolSettingsKind, func(value []byte) error {
|
||||
var settings toolSettings
|
||||
if err := json.Unmarshal(value, &settings); err != nil {
|
||||
return err
|
||||
@@ -81,7 +153,7 @@ func (c *Client) runtimeToolConfig(ctx context.Context, versions map[string]int6
|
||||
}); err != nil {
|
||||
return config.Config{}, nil, err
|
||||
}
|
||||
if err := c.queryInto(ctx, versions, PluginSettingsKind, func(value []byte) error {
|
||||
if err := layerResource(ctx, reader, versions, PluginSettingsKind, func(value []byte) error {
|
||||
var settings pluginSettings
|
||||
if err := json.Unmarshal(value, &settings); err != nil {
|
||||
return err
|
||||
@@ -91,15 +163,15 @@ func (c *Client) runtimeToolConfig(ctx context.Context, versions map[string]int6
|
||||
}); err != nil {
|
||||
return config.Config{}, nil, err
|
||||
}
|
||||
if err := c.queryJSONInto(ctx, versions, ContainerRuntimePoliciesKind, &out.Containers); err != nil {
|
||||
if err := layerResourceJSON(ctx, reader, versions, ContainerRuntimePoliciesKind, &out.Containers); err != nil {
|
||||
return config.Config{}, nil, err
|
||||
}
|
||||
return out, versions, nil
|
||||
}
|
||||
|
||||
func (c *Client) RuntimeChatConfig(ctx context.Context, base config.ChatConfig) (config.ChatConfig, int64, error) {
|
||||
func runtimeChatConfigFrom(ctx context.Context, reader resourceReader, base config.ChatConfig) (config.ChatConfig, int64, error) {
|
||||
out := base
|
||||
version, err := c.query(ctx, ChannelSettingsKind, func(value []byte) error {
|
||||
version, err := reader.query(ctx, ChannelSettingsKind, func(value []byte) error {
|
||||
var settings channelSettings
|
||||
if err := json.Unmarshal(value, &settings); err != nil {
|
||||
return err
|
||||
@@ -117,10 +189,10 @@ func (c *Client) RuntimeChatConfig(ctx context.Context, base config.ChatConfig)
|
||||
return out, version, err
|
||||
}
|
||||
|
||||
// queryInto decodes a resource and records the version it came from, so
|
||||
// RuntimeConfig ends up holding the version of every kind it layered in.
|
||||
func (c *Client) queryInto(ctx context.Context, versions map[string]int64, kind string, decode func([]byte) error) error {
|
||||
version, err := c.query(ctx, kind, decode)
|
||||
// layerResource decodes a resource and records the version it came from, so the
|
||||
// layering ends up holding the version of every kind it applied.
|
||||
func layerResource(ctx context.Context, reader resourceReader, versions map[string]int64, kind string, decode func([]byte) error) error {
|
||||
version, err := reader.query(ctx, kind, decode)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -128,8 +200,8 @@ func (c *Client) queryInto(ctx context.Context, versions map[string]int64, kind
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) queryJSONInto(ctx context.Context, versions map[string]int64, kind string, target any) error {
|
||||
return c.queryInto(ctx, versions, kind, func(value []byte) error { return json.Unmarshal(value, target) })
|
||||
func layerResourceJSON(ctx context.Context, reader resourceReader, versions map[string]int64, kind string, target any) error {
|
||||
return layerResource(ctx, reader, versions, kind, func(value []byte) error { return json.Unmarshal(value, target) })
|
||||
}
|
||||
|
||||
// query returns the version of the resource it decoded, so callers that layer
|
||||
|
||||
@@ -382,6 +382,13 @@ esac
|
||||
`)
|
||||
writeFakeCommand(t, fakeDir, "go", `
|
||||
printf '%s\n' "go $*" >> "$ARCHIE_TEST_CALLS"
|
||||
# The pre-flight task-database backup runs through the release's own
|
||||
# archie-state-store subcommand, so a release whose recovery path cannot run
|
||||
# must refuse the update.
|
||||
if [ "$1" = "run" ]; then
|
||||
[ "${ARCHIE_TEST_GO_FAILS_RUN:-}" = "1" ] && exit 1
|
||||
exit 0
|
||||
fi
|
||||
out=""
|
||||
while [ "$#" -gt 0 ]; do
|
||||
if [ "$1" = "-o" ]; then out="$2"; break; fi
|
||||
@@ -745,6 +752,10 @@ func TestUpdateWatchdogCyclesEveryUnitInDependencyOrder(t *testing.T) {
|
||||
// value up produces an empty backup, and a rollback then restores that empty
|
||||
// file over the database and deletes its -wal/-shm. A rollback that destroys
|
||||
// the task store is worse than no rollback.
|
||||
//
|
||||
// The snapshot is taken by the release's own recovery subcommand rather than
|
||||
// by hand-rolled shell, so the installer and an operator run the same code and
|
||||
// the two cannot drift apart.
|
||||
func TestUpdateInstallBacksUpTheSiblingTaskDatabase(t *testing.T) {
|
||||
work := t.TempDir()
|
||||
configured := filepath.Join(work, "archie.db")
|
||||
@@ -764,7 +775,8 @@ func TestUpdateInstallBacksUpTheSiblingTaskDatabase(t *testing.T) {
|
||||
"ARCHIE_TASK_DB_BACKUP": backup,
|
||||
}, "db_path = \""+configured+"\"\n\n[containers]\nimage = 'registry.example/archie-agent:stable'\n", false)
|
||||
|
||||
assertCallContains(t, calls, "sqlite3", realStore, ".backup")
|
||||
assertCallContains(t, calls, "go run ./cmd/archie-state-store backup", "-db "+realStore, "-out "+backup)
|
||||
assertCallAbsent(t, calls, "sqlite3")
|
||||
assertCallContains(t, calls, "systemd-run", "--setenv=ARCHIE_TASK_DB_PATH="+realStore)
|
||||
// The zero-byte placeholder is not the store, and must never be the thing
|
||||
// handed to the watchdog as ARCHIE_TASK_DB_PATH.
|
||||
@@ -793,9 +805,9 @@ func TestUpdateInstallIgnoresDbPathFromOtherSections(t *testing.T) {
|
||||
"ARCHIE_TASK_DB_BACKUP": backup,
|
||||
}, "db_path = \""+configured+"\"\n\n[indexing]\ndb_path = \""+indexStore+"\"\n\n[containers]\nimage = 'registry.example/archie-agent:stable'\n", false)
|
||||
|
||||
assertCallContains(t, calls, "sqlite3", realStore)
|
||||
assertCallContains(t, calls, "go run ./cmd/archie-state-store backup", "-db "+realStore)
|
||||
for _, call := range calls {
|
||||
if strings.Contains(call, "sqlite3") && strings.Contains(call, indexStore) {
|
||||
if strings.Contains(call, "go run ./cmd/archie-state-store backup") && strings.Contains(call, indexStore) {
|
||||
t.Errorf("backed up the indexing database %s: %q", indexStore, call)
|
||||
}
|
||||
}
|
||||
@@ -815,7 +827,7 @@ func TestUpdateInstallRefusesWhenTaskDatabaseCannotBeBackedUp(t *testing.T) {
|
||||
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0",
|
||||
"ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0",
|
||||
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0",
|
||||
"ARCHIE_TEST_SQLITE3_FAILS": "1",
|
||||
"ARCHIE_TEST_GO_FAILS_RUN": "1",
|
||||
}, "db_path = \""+configured+"\"\n\n[containers]\nimage = 'registry.example/archie-agent:stable'\n", true)
|
||||
|
||||
if !strings.Contains(output, "task database") {
|
||||
|
||||
@@ -0,0 +1,265 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// This file is the offline half of the store's contract with its owner: the
|
||||
// operations the State Store process cannot perform on itself, because they
|
||||
// read or replace the very file it has open. They are what
|
||||
// docs/prds/runtime-control-plane.md ("Bootstrap, migration, and recovery")
|
||||
// asks for -- back up, restore and validate the database without archied or
|
||||
// the Web UI -- and every one of them works on the file directly.
|
||||
//
|
||||
// The read-only operations never take the ownership lock, because they are
|
||||
// what an operator runs against a serving store. The one that replaces the
|
||||
// file does, because a rewrite under a running server is the only way an
|
||||
// offline command can destroy a task store.
|
||||
|
||||
// OpenReadOnly opens an existing store for reading only: no schema is created
|
||||
// and no migration runs. That matters for recovery -- the snapshot an update
|
||||
// takes has to stay exactly what the release that wrote it expects, and
|
||||
// migrating a copy during verification would hand a rollback a schema its
|
||||
// binary refuses.
|
||||
func OpenReadOnly(ctx context.Context, path string) (*Store, error) {
|
||||
if err := RequireDatabase(path); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
db, err := sql.Open("sqlite", readonlyDSN(path))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
db.SetMaxOpenConns(1)
|
||||
// sql.Open is lazy: the first statement is where a file that is not a
|
||||
// database, a missing WAL sidecar or an unreadable path surfaces.
|
||||
var tables int
|
||||
if err := db.QueryRowContext(ctx, `SELECT count(*) FROM sqlite_master`).Scan(&tables); err != nil {
|
||||
return nil, errors.Join(fmt.Errorf("read task database %s: %w", path, err), db.Close())
|
||||
}
|
||||
return &Store{db: db}, nil
|
||||
}
|
||||
|
||||
// readonlyDSN reads the store without the write-ahead log pragma the serving
|
||||
// process sets: a reader has no business changing the database's journal mode,
|
||||
// and the read-only flag makes an accidental write an error rather than a
|
||||
// silent modification of a file someone else owns.
|
||||
func readonlyDSN(path string) string {
|
||||
separator := "?"
|
||||
if strings.HasSuffix(path, "?") || strings.HasSuffix(path, "&") {
|
||||
separator = ""
|
||||
} else if strings.Contains(path, "?") {
|
||||
separator = "&"
|
||||
}
|
||||
return path + separator + "mode=ro&_pragma=busy_timeout(5000)"
|
||||
}
|
||||
|
||||
// StoresResources reports whether the store's file carries the control-plane
|
||||
// resources table. A store written before the control plane existed has none
|
||||
// and holds no stored settings to validate; the serving process creates the
|
||||
// table on its next start, so an offline check must tell that apart from
|
||||
// resources it cannot read.
|
||||
func (s *Store) StoresResources(ctx context.Context) (bool, error) {
|
||||
var tables int
|
||||
if err := s.db.QueryRowContext(ctx, `SELECT count(*) FROM sqlite_master WHERE type='table' AND name='resources'`).Scan(&tables); err != nil {
|
||||
return false, fmt.Errorf("inspect store schema: %w", err)
|
||||
}
|
||||
return tables > 0, nil
|
||||
}
|
||||
|
||||
// RequireDatabase reports whether path is an existing store file. A writer
|
||||
// that must not create the file it fails to find asks this first: a recovery
|
||||
// command pointed at a mistyped path has to refuse, not leave a fresh empty
|
||||
// database where an operator expected theirs.
|
||||
func RequireDatabase(path string) error {
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
return fmt.Errorf("task database %s: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ValidateFile checks an existing store the way the serving process meets it
|
||||
// on startup and reports the schema version it carries: the file is a SQLite
|
||||
// database this binary can read, it is not corrupt, and its schema is not
|
||||
// newer than this binary supports.
|
||||
func ValidateFile(ctx context.Context, path string) (int, error) {
|
||||
st, err := OpenReadOnly(ctx, path)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer func() { _ = st.Close() }()
|
||||
return st.check(ctx)
|
||||
}
|
||||
|
||||
// check runs the file-level checks. integrity_check returns one row per
|
||||
// problem, or the single row "ok"; reading the first row is therefore the
|
||||
// verdict rather than a sample.
|
||||
func (s *Store) check(ctx context.Context) (int, error) {
|
||||
var result string
|
||||
if err := s.db.QueryRowContext(ctx, `PRAGMA integrity_check`).Scan(&result); err != nil {
|
||||
return 0, fmt.Errorf("integrity check: %w", err)
|
||||
}
|
||||
if result != "ok" {
|
||||
return 0, fmt.Errorf("integrity check: %s", result)
|
||||
}
|
||||
var version int
|
||||
if err := s.db.QueryRowContext(ctx, `PRAGMA user_version`).Scan(&version); err != nil {
|
||||
return 0, fmt.Errorf("read schema version: %w", err)
|
||||
}
|
||||
if version > taskSchemaVersion {
|
||||
return 0, fmt.Errorf("database schema version %d is newer than supported version %d", version, taskSchemaVersion)
|
||||
}
|
||||
return version, nil
|
||||
}
|
||||
|
||||
// Backup writes a transactionally consistent snapshot of database to snapshot,
|
||||
// using SQLite's own copy, so a store that is being written to cannot produce
|
||||
// a torn file.
|
||||
//
|
||||
// It deliberately does not take the ownership lock. The update installer runs
|
||||
// inside the process it is updating and cannot stop the State Store, so the
|
||||
// snapshot that makes a failed update reversible is taken against a serving
|
||||
// store; a backup that insisted on an exclusively owned file would break the
|
||||
// one caller that exists. The snapshot is written beside the destination and
|
||||
// renamed into place, so an interrupted backup never replaces a snapshot that
|
||||
// was good with one that is not.
|
||||
func Backup(ctx context.Context, database, snapshot string) error {
|
||||
if err := RequireDatabase(database); err != nil {
|
||||
return err
|
||||
}
|
||||
// The exemption from the ownership lock below is only sound while backup
|
||||
// never replaces the database. With -out naming the database it would
|
||||
// rename the snapshot over a file a serving State Store may still have
|
||||
// open, orphaning the WAL of the file it just unlinked -- so it is refused
|
||||
// exactly as Restore refuses it.
|
||||
if same, err := sameFile(database, snapshot); err != nil {
|
||||
return err
|
||||
} else if same {
|
||||
return fmt.Errorf("snapshot %s is the database itself", snapshot)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(snapshot), 0o755); err != nil {
|
||||
return fmt.Errorf("create snapshot directory: %w", err)
|
||||
}
|
||||
db, err := sql.Open("sqlite", sqliteDSN(database))
|
||||
if err != nil {
|
||||
return fmt.Errorf("open task database %s: %w", database, err)
|
||||
}
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
staging := snapshot + ".tmp"
|
||||
if err := os.Remove(staging); err != nil && !os.IsNotExist(err) {
|
||||
return fmt.Errorf("clear staging snapshot %s: %w", staging, err)
|
||||
}
|
||||
if _, err := db.ExecContext(ctx, `VACUUM INTO ?`, staging); err != nil {
|
||||
return errors.Join(fmt.Errorf("snapshot %s: %w", database, err), os.Remove(staging))
|
||||
}
|
||||
// A snapshot that is not itself a usable store is worse than no snapshot:
|
||||
// the update path treats its existence as proof that a rollback is
|
||||
// possible.
|
||||
if _, err := ValidateFile(ctx, staging); err != nil {
|
||||
return errors.Join(fmt.Errorf("snapshot is not a usable store: %w", err), os.Remove(staging))
|
||||
}
|
||||
if err := os.Rename(staging, snapshot); err != nil {
|
||||
return errors.Join(fmt.Errorf("publish snapshot %s: %w", snapshot, err), os.Remove(staging))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Restore replaces database with the snapshot at snapshot. The snapshot is
|
||||
// verified before anything is destroyed, so a file that is not a usable store
|
||||
// cannot take the database down with it, and the database file itself is
|
||||
// replaced by a rename, so a failure leaves either the old file or the new one
|
||||
// rather than a mixture. The replaced file's -wal/-shm are removed first, which
|
||||
// the comment at that step explains: between the two calls the database is
|
||||
// merely incomplete, and re-running the command completes it.
|
||||
//
|
||||
// It holds the ownership lock for its duration: rewriting a database a running
|
||||
// State Store still has open would leave that process writing to an unlinked
|
||||
// inode while the store served a file nobody owns.
|
||||
func Restore(ctx context.Context, database, snapshot string) error {
|
||||
if _, err := os.Stat(filepath.Dir(database)); err != nil {
|
||||
return fmt.Errorf("restore destination %s: %w", database, err)
|
||||
}
|
||||
if same, err := sameFile(database, snapshot); err != nil {
|
||||
return err
|
||||
} else if same {
|
||||
return fmt.Errorf("restore snapshot %s is the database itself", snapshot)
|
||||
}
|
||||
ownership, err := AcquireOwnership(database)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = ownership.Release() }()
|
||||
|
||||
if _, err := ValidateFile(ctx, snapshot); err != nil {
|
||||
return fmt.Errorf("snapshot is not a usable store: %w", err)
|
||||
}
|
||||
staging := database + ".restore.tmp"
|
||||
if err := copyFile(snapshot, staging); err != nil {
|
||||
return errors.Join(err, os.Remove(staging))
|
||||
} // The replaced database's write-ahead log belongs to the file that is
|
||||
// going away. It goes first: between the two calls the database is merely
|
||||
// incomplete, whereas a WAL left beside the file it does not describe is
|
||||
// the state SQLite cannot recover from.
|
||||
for _, sidecar := range []string{"-wal", "-shm"} {
|
||||
if err := os.Remove(database + sidecar); err != nil && !os.IsNotExist(err) {
|
||||
return errors.Join(fmt.Errorf("remove %s%s: %w", database, sidecar, err), os.Remove(staging))
|
||||
}
|
||||
}
|
||||
if err := os.Rename(staging, database); err != nil {
|
||||
return errors.Join(fmt.Errorf("publish restored database %s: %w", database, err), os.Remove(staging))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sameFile(a, b string) (bool, error) {
|
||||
first, err := os.Stat(a)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
second, err := os.Stat(b)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return os.SameFile(first, second), nil
|
||||
}
|
||||
|
||||
// copyFile writes source over a new file at path, flushed to disk before it is
|
||||
// returned: the rename that publishes it must not reach the directory entries
|
||||
// ahead of the bytes they point at. The snapshot's own permissions are kept, so
|
||||
// a restore does not silently change who may read the operator's database.
|
||||
func copyFile(source, path string) (retErr error) {
|
||||
info, err := os.Stat(source)
|
||||
if err != nil {
|
||||
return fmt.Errorf("stat snapshot %s: %w", source, err)
|
||||
}
|
||||
in, err := os.Open(source)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open snapshot %s: %w", source, err)
|
||||
}
|
||||
defer func() { retErr = errors.Join(retErr, in.Close()) }()
|
||||
out, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, info.Mode().Perm())
|
||||
if err != nil {
|
||||
return fmt.Errorf("create %s: %w", path, err)
|
||||
}
|
||||
defer func() { retErr = errors.Join(retErr, out.Close()) }()
|
||||
if _, err := io.Copy(out, in); err != nil {
|
||||
return fmt.Errorf("write %s: %w", path, err)
|
||||
}
|
||||
if err := out.Sync(); err != nil {
|
||||
return fmt.Errorf("sync %s: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Ownership is what makes "the State Store is stopped" a fact an offline
|
||||
// command can check, so the primitive itself is asserted here rather than only
|
||||
// through the commands that use it: exclusivity, release, and per-database
|
||||
// independence.
|
||||
func TestOwnershipIsExclusivePerDatabase(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
first := filepath.Join(dir, "archie.db-tasks.sqlite")
|
||||
second := filepath.Join(dir, "other.db-tasks.sqlite")
|
||||
|
||||
held, err := AcquireOwnership(first)
|
||||
if err != nil {
|
||||
t.Fatalf("first AcquireOwnership: %v", err)
|
||||
}
|
||||
if _, err := AcquireOwnership(first); !errors.Is(err, ErrStoreOwned) {
|
||||
t.Fatalf("second AcquireOwnership on the same database = %v, want ErrStoreOwned", err)
|
||||
}
|
||||
// A different database is a different claim: one process owning its store
|
||||
// must not stop another from owning its own.
|
||||
elsewhere, err := AcquireOwnership(second)
|
||||
if err != nil {
|
||||
t.Fatalf("AcquireOwnership on an unrelated database: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(first + ".lock"); err != nil {
|
||||
t.Fatalf("ownership left no sidecar to lock: %v", err)
|
||||
}
|
||||
|
||||
if err := held.Release(); err != nil {
|
||||
t.Fatalf("Release: %v", err)
|
||||
}
|
||||
reacquired, err := AcquireOwnership(first)
|
||||
if err != nil {
|
||||
t.Fatalf("AcquireOwnership after Release: %v", err)
|
||||
}
|
||||
if err := errors.Join(reacquired.Release(), elsewhere.Release()); err != nil {
|
||||
t.Fatalf("release on cleanup: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The snapshot an update takes is the state a rollback restores, and it is
|
||||
// restored by the release that wrote it. Verifying the copy must therefore not
|
||||
// migrate it: a snapshot silently upgraded to this binary's schema would be
|
||||
// refused by the older binary it exists to serve.
|
||||
func TestBackupSnapshotKeepsTheSchemaVersionItCopied(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
database := filepath.Join(dir, "archie.db-tasks.sqlite")
|
||||
snapshot := filepath.Join(dir, "snapshot.sqlite")
|
||||
st, err := Open(t.Context(), database)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := st.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// An older release's file: the schema version it wrote, which this binary
|
||||
// is expected to migrate when the store is next opened for serving.
|
||||
if err := setUserVersion(t.Context(), database, taskSchemaVersion-1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := Backup(t.Context(), database, snapshot); err != nil {
|
||||
t.Fatalf("Backup: %v", err)
|
||||
}
|
||||
version, err := ValidateFile(t.Context(), snapshot)
|
||||
if err != nil {
|
||||
t.Fatalf("ValidateFile on the snapshot: %v", err)
|
||||
}
|
||||
if version != taskSchemaVersion-1 {
|
||||
t.Fatalf("snapshot schema version = %d, want the %d it copied", version, taskSchemaVersion-1)
|
||||
}
|
||||
}
|
||||
|
||||
func setUserVersion(ctx context.Context, path string, version int) error {
|
||||
db, err := sql.Open("sqlite", path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = db.Close() }()
|
||||
_, err = db.ExecContext(ctx, fmt.Sprintf(`PRAGMA user_version = %d`, version))
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// ErrStoreOwned is returned when a task database is already owned by another
|
||||
// process. Ownership is what makes "the State Store is stopped" a fact an
|
||||
// offline command can check rather than a rule the operator is trusted to
|
||||
// follow: overwriting a database a running server still holds leaves it
|
||||
// writing to an unlinked inode while the store serves a file nobody owns.
|
||||
var ErrStoreOwned = errors.New("task database is owned by another process")
|
||||
|
||||
// Ownership is one process's exclusive claim over one task database, held for
|
||||
// as long as that process owns the file. The serving State Store takes it for
|
||||
// its whole life; the offline recovery commands take it for the duration of an
|
||||
// operation that rewrites the file.
|
||||
//
|
||||
// The claim lives in a sidecar file beside the database rather than on the
|
||||
// database inode itself, so it survives the rename a restore performs: after
|
||||
// the swap the sidecar is still the one path every process locks.
|
||||
type Ownership struct {
|
||||
f *os.File
|
||||
}
|
||||
|
||||
// AcquireOwnership takes the store's ownership lock without blocking. A
|
||||
// database another process owns comes back as ErrStoreOwned, never as a wait:
|
||||
// a process that must not proceed has to hear so, and a blocking lock would
|
||||
// turn a mistyped command into a hung terminal.
|
||||
func AcquireOwnership(path string) (*Ownership, error) {
|
||||
lockPath := ownershipLockPath(path)
|
||||
if err := os.MkdirAll(filepath.Dir(lockPath), 0o755); err != nil {
|
||||
return nil, fmt.Errorf("create store directory for %s: %w", lockPath, err)
|
||||
}
|
||||
f, err := os.OpenFile(lockPath, os.O_RDWR|os.O_CREATE, 0o644)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open store lock %s: %w", lockPath, err)
|
||||
}
|
||||
taken, err := lockFile(f)
|
||||
if err != nil {
|
||||
return nil, errors.Join(fmt.Errorf("lock %s: %w", lockPath, err), f.Close())
|
||||
}
|
||||
if !taken {
|
||||
return nil, errors.Join(fmt.Errorf("%s: %w", path, ErrStoreOwned), f.Close())
|
||||
}
|
||||
return &Ownership{f: f}, nil
|
||||
}
|
||||
|
||||
// Release drops the claim and closes the sidecar. Closing the descriptor alone
|
||||
// would release it too; the explicit unlock keeps the intent readable and
|
||||
// makes Release idempotent.
|
||||
func (o *Ownership) Release() error {
|
||||
if o == nil || o.f == nil {
|
||||
return nil
|
||||
}
|
||||
err := unlockFile(o.f)
|
||||
f := o.f
|
||||
o.f = nil
|
||||
return errors.Join(err, f.Close())
|
||||
}
|
||||
|
||||
// ownershipLockPath is the sidecar every owner of path locks. It is derived
|
||||
// from the database path, so the State Store process and an offline command
|
||||
// that were given the same database agree on it without configuration.
|
||||
func ownershipLockPath(path string) string { return path + ".lock" }
|
||||
@@ -0,0 +1,30 @@
|
||||
//go:build unix
|
||||
|
||||
package store
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// lockFile takes a non-blocking exclusive flock on f. The boolean reports
|
||||
// whether it was taken: a lock another process holds (EWOULDBLOCK) is a normal
|
||||
// answer to "is the State Store running?", not an error.
|
||||
func lockFile(f *os.File) (bool, error) {
|
||||
err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
|
||||
if errors.Is(err, syscall.EWOULDBLOCK) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func unlockFile(f *os.File) error {
|
||||
if err := syscall.Flock(int(f.Fd()), syscall.LOCK_UN); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
//go:build windows
|
||||
|
||||
// flock is a POSIX primitive and Windows is not a supported deployment surface
|
||||
// (CLAUDE.md's deployment model is Linux and macOS), so this file fails closed:
|
||||
// every lock attempt reports an error rather than silently degrading to "no
|
||||
// lock". An offline command that cannot prove it owns the database must refuse
|
||||
// to rewrite it, which is the only safe reading of an unavailable primitive.
|
||||
//
|
||||
// If a Windows deployment ever becomes supported, swap this for a real
|
||||
// LockFileEx/UnlockFileEx pair; nothing else in the package needs to change.
|
||||
|
||||
package store
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
)
|
||||
|
||||
func lockFile(*os.File) (bool, error) {
|
||||
return false, errors.New("store ownership locking requires a POSIX flock, which this platform does not provide")
|
||||
}
|
||||
|
||||
func unlockFile(*os.File) error { return nil }
|
||||
@@ -183,10 +183,18 @@ fi
|
||||
# that cannot take one must not start: once binaries are replaced, a failure
|
||||
# leaves the previous release unable to read the store it just migrated. This
|
||||
# runs ahead of the adapter refresh below so a refusal changes nothing at all.
|
||||
#
|
||||
# The snapshot is taken by the recovery subcommand of the release being
|
||||
# installed, from its own checkout, rather than by hand-rolled sqlite3 shell:
|
||||
# the operator running `archie-state-store backup` by hand and the installer
|
||||
# are then the same implementation, and the two cannot drift. The installed
|
||||
# binary is the previous release, which may predate the subcommand, so it
|
||||
# cannot be the one asked -- and a release whose recovery path will not build is a
|
||||
# release that cannot back up, which is exactly what this pre-flight refuses.
|
||||
if [[ ",$components," == *,daemon,* ]] && [ -n "$TASK_DB_PATH" ] && [ -f "$TASK_DB_PATH" ]; then
|
||||
echo "==> backing up task database for schema-safe rollback"
|
||||
mkdir -p "$(dirname "$TASK_DB_BACKUP")"
|
||||
if ! sqlite3 "$TASK_DB_PATH" ".backup '$TASK_DB_BACKUP'"; then
|
||||
if ! go run ./cmd/archie-state-store backup -db "$TASK_DB_PATH" -out "$TASK_DB_BACKUP"; then
|
||||
echo "refusing to install $GW: could not back up the task database at $TASK_DB_PATH, so a failed update could not be rolled back" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
Reference in new issue
Block a user