feat(statestore): offline backup, restore, validate and rollback
archie-state-store gains the offline recovery commands
docs/prds/runtime-control-plane.md ("Bootstrap, migration, and recovery")
asks for. They work on the task database file directly -- <db_path>-tasks.sqlite,
not db_path -- and need neither archied nor the Web UI, which matters because
the failure they exist for is the one that stops archied starting:
backup -db F -out F consistent snapshot (VACUUM INTO), verified as a
usable store before it is published; safe against
a serving store
restore -db F -from F replace the database with a snapshot, dropping the
replaced file's -wal/-shm
validate -db F readable, not corrupt, schema no newer than this
binary, every stored resource decoding under the
definition that owns it
rollback -db F -kind K replay an earlier revision of a stored resource
[-revision N] through the ordinary replace, recorded as a new
revision
No rollback RPC is added and none is needed: a revision carries its own value,
so restoring one is the same replace the dashboard and the channels already
use, by hand, with the store stopped.
"The State Store is stopped" is checked rather than assumed. The serving
process now takes an exclusive flock on <db>.lock for its whole life
(store.AcquireOwnership), and restore/rollback refuse with
`task database is owned by another process` while it holds it. backup is the
deliberate exception: the update installer runs inside the process it is
replacing and cannot stop the State Store.
scripts/archie-update-install takes its pre-flight snapshot through
`go run ./cmd/archie-state-store backup` from the release being installed
instead of its own sqlite3 shell, so the installer and an operator run one
implementation. The watchdog's refusal semantics are unchanged: an update that
cannot back up still must not start.
Docs: safe-change-and-recovery.md's "Recovery: not yet built" now describes
what exists.
This commit is contained in:
14 files changed
+1417
-18
No files matched your search
@@ -15,6 +15,7 @@ import (
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/app/archied"
|
||||
@@ -23,6 +24,13 @@ import (
|
||||
func main() { os.Exit(run()) }
|
||||
|
||||
func run() int {
|
||||
// A leading non-flag argument selects a recovery subcommand; anything else,
|
||||
// including no arguments at all, serves the State Store contract. An
|
||||
// unrecognised word is refused rather than ignored, so a mistyped recovery
|
||||
// command never leaves an operator watching a server start instead.
|
||||
if args := os.Args[1:]; len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||
return runRecovery(args, os.Stdout, os.Stderr)
|
||||
}
|
||||
base, err := os.UserConfigDir()
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
@@ -34,6 +42,10 @@ func run() int {
|
||||
flag.StringVar(&options.Listen, "listen", "", "state store gRPC listen address (defaults to [services.state].listen, else 127.0.0.1:9090)")
|
||||
flag.StringVar(&options.Token, "token", "", "bearer token required for a non-loopback listener (defaults to [services.state].target_token)")
|
||||
flag.StringVar(&options.ReadyAddr, "ready-addr", "", "optional readiness HTTP listen address (e.g. 127.0.0.1:9091)")
|
||||
flag.Usage = func() {
|
||||
fmt.Fprintf(flag.CommandLine.Output(), "usage: archie-state-store [flags] # serve the State Store gRPC contract\n\noffline recovery: archie-state-store <backup|restore|validate|rollback> -h\n\n")
|
||||
flag.PrintDefaults()
|
||||
}
|
||||
flag.Parse()
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/app/archied"
|
||||
)
|
||||
|
||||
// recoveryUsage is the whole flag surface of the offline recovery commands.
|
||||
// They are subcommands of this binary rather than a recovery binary of their
|
||||
// own: offline database maintenance belongs to the process that owns the file
|
||||
// and its path layout (docs/architecture/organisation.md#process-boundaries).
|
||||
const recoveryUsage = `usage: archie-state-store <command> [flags]
|
||||
|
||||
With no command, archie-state-store serves the State Store gRPC contract.
|
||||
|
||||
The recovery commands operate on the task database file directly -- the
|
||||
configured db_path with "-tasks.sqlite" appended -- and are how an operator
|
||||
recovers a database the control plane will not start on. Stop the State Store
|
||||
first: restore and rollback rewrite the file and refuse while another process
|
||||
owns it, while backup takes a consistent snapshot of a serving store.
|
||||
|
||||
backup -db FILE -out FILE write a snapshot of the database
|
||||
restore -db FILE -from FILE replace the database with a snapshot
|
||||
validate -db FILE check the database the way boot does
|
||||
rollback -db FILE -kind KIND replay an earlier revision of a stored
|
||||
[-revision N] resource through the ordinary replace
|
||||
`
|
||||
|
||||
// runRecovery parses and performs one offline recovery command, reporting the
|
||||
// operation's summary on stdout. Exit codes follow the serve path: 0 succeeded,
|
||||
// 1 the operation failed, 2 the command line was wrong.
|
||||
func runRecovery(args []string, stdout, stderr io.Writer) int {
|
||||
if len(args) == 0 {
|
||||
fmt.Fprint(stderr, recoveryUsage)
|
||||
return 2
|
||||
}
|
||||
command := args[0]
|
||||
options := archied.StateStoreRecoveryOptions{Operation: command}
|
||||
switch command {
|
||||
case archied.RecoveryBackup, archied.RecoveryRestore, archied.RecoveryValidate, archied.RecoveryRollback:
|
||||
default:
|
||||
fmt.Fprintf(stderr, "archie-state-store: unknown command %q\n\n", command)
|
||||
fmt.Fprint(stderr, recoveryUsage)
|
||||
return 2
|
||||
}
|
||||
|
||||
flags := flag.NewFlagSet("archie-state-store "+command, flag.ContinueOnError)
|
||||
flags.SetOutput(stderr)
|
||||
flags.Usage = func() { fmt.Fprint(stderr, recoveryUsage) }
|
||||
flags.StringVar(&options.DB, "db", "", "task database file the State Store owns (<db_path>-tasks.sqlite)")
|
||||
flags.StringVar(&options.Out, "out", "", "snapshot file backup writes")
|
||||
flags.StringVar(&options.From, "from", "", "snapshot file restore reads")
|
||||
flags.StringVar(&options.Kind, "kind", "", "control-plane resource kind rollback replays")
|
||||
flags.Int64Var(&options.Revision, "revision", 0, "revision rollback replays (default: the newest one older than the current value)")
|
||||
if err := flags.Parse(args[1:]); err != nil {
|
||||
return 2
|
||||
}
|
||||
if flags.NArg() > 0 {
|
||||
fmt.Fprintf(stderr, "archie-state-store %s: unexpected argument %q\n", command, flags.Arg(0))
|
||||
return 2
|
||||
}
|
||||
|
||||
summary, err := archied.RunStateStoreRecovery(context.Background(), options)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "archie-state-store %s: %v\n", command, err)
|
||||
return 1
|
||||
}
|
||||
fmt.Fprintln(stdout, summary)
|
||||
return 0
|
||||
}
|
||||
@@ -0,0 +1,544 @@
|
||||
// Package main tests the offline recovery subcommands of archie-state-store:
|
||||
// backup, restore, validate and rollback against the task database file
|
||||
// directly. They are the operator's only path back when the control plane's
|
||||
// stored settings will not validate, because the daemon fails closed: the
|
||||
// in-band remedy (replay an earlier revision while the State Store is up) is
|
||||
// unavailable in exactly the case that needs it.
|
||||
//
|
||||
// The tests drive the subcommands the way an operator does -- through
|
||||
// runRecovery, against real files on disk -- rather than through the internal
|
||||
// helpers, so the flag surface and the exit codes are part of what is verified.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/app/archied"
|
||||
"github.com/samcharles93/archie-core/internal/app/controlplane"
|
||||
"github.com/samcharles93/archie-core/internal/store"
|
||||
)
|
||||
|
||||
// runRecoveryCmd runs one subcommand the way main does and returns its exit
|
||||
// code with both streams captured.
|
||||
func runRecoveryCmd(t *testing.T, args ...string) (int, string, string) {
|
||||
t.Helper()
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := runRecovery(args, &stdout, &stderr)
|
||||
return code, stdout.String(), stderr.String()
|
||||
}
|
||||
|
||||
// openTaskStore opens the task database the subcommands operate on. It does not
|
||||
// take the ownership lock: only the serving process does that, and a test that
|
||||
// wants a "live" store holds it explicitly with holdStoreOwnership.
|
||||
func openTaskStore(t *testing.T, path string) *store.Store {
|
||||
t.Helper()
|
||||
st, err := store.Open(t.Context(), path)
|
||||
if err != nil {
|
||||
t.Fatalf("open task store %s: %v", path, err)
|
||||
}
|
||||
t.Cleanup(func() { _ = st.Close() })
|
||||
return st
|
||||
}
|
||||
|
||||
// holdStoreOwnership takes the ownership lock the running State Store holds for
|
||||
// its whole life, so a test can prove a command refuses to touch a live store.
|
||||
func holdStoreOwnership(t *testing.T, path string) {
|
||||
t.Helper()
|
||||
ownership, err := store.AcquireOwnership(path)
|
||||
if err != nil {
|
||||
t.Fatalf("acquire store ownership: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = ownership.Release() })
|
||||
}
|
||||
|
||||
// seedTask writes one task so a snapshot can be distinguished from the live
|
||||
// database by what it contains, not merely by its existence.
|
||||
func seedTask(t *testing.T, st *store.Store, title string) {
|
||||
t.Helper()
|
||||
if _, err := st.EnqueueChatTask(t.Context(), "acme", "widget", title, "body", "implement", ""); err != nil {
|
||||
t.Fatalf("seed task %q: %v", title, err)
|
||||
}
|
||||
}
|
||||
|
||||
func taskTitles(t *testing.T, path string) []string {
|
||||
t.Helper()
|
||||
st := openTaskStore(t, path)
|
||||
tasks, err := st.Tasks(t.Context(), 100)
|
||||
if err != nil {
|
||||
t.Fatalf("list tasks in %s: %v", path, err)
|
||||
}
|
||||
titles := make([]string, 0, len(tasks))
|
||||
for _, task := range tasks {
|
||||
titles = append(titles, task.Title)
|
||||
}
|
||||
return titles
|
||||
}
|
||||
|
||||
// backupPath is the snapshot every restore test restores from.
|
||||
func backupPath(t *testing.T, dir string) string { return filepath.Join(dir, "snapshot.sqlite") }
|
||||
|
||||
// backup takes the snapshot the caller then restores, through the subcommand
|
||||
// under test rather than through the store directly, so a broken backup fails
|
||||
// the restore test too instead of silently passing an empty file.
|
||||
func backup(t *testing.T, db, out string) {
|
||||
t.Helper()
|
||||
code, _, stderr := runRecoveryCmd(t, "backup", "-db", db, "-out", out)
|
||||
if code != 0 {
|
||||
t.Fatalf("backup exited %d: %s", code, stderr)
|
||||
}
|
||||
}
|
||||
|
||||
// The update installer cannot stop the process that runs it, so it backs the
|
||||
// task database up while the State Store is serving. VACUUM INTO is safe
|
||||
// against a live writer; a backup that insisted on an exclusively-owned file
|
||||
// would break the one caller it has, and the update would refuse to start.
|
||||
func TestRecoveryBackupSnapshotsAStoreItsStateStoreIsServing(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
db := filepath.Join(dir, "archie.db-tasks.sqlite")
|
||||
out := backupPath(t, dir)
|
||||
seedTask(t, openTaskStore(t, db), "before")
|
||||
holdStoreOwnership(t, db)
|
||||
if err := os.WriteFile(out, []byte("stale snapshot"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
code, stdout, stderr := runRecoveryCmd(t, "backup", "-db", db, "-out", out)
|
||||
if code != 0 {
|
||||
t.Fatalf("backup exited %d: %s", code, stderr)
|
||||
}
|
||||
if !strings.Contains(stdout, out) || !strings.Contains(stdout, db) {
|
||||
t.Errorf("backup must report the database and the snapshot it wrote; stdout = %q", stdout)
|
||||
}
|
||||
// The stale file was replaced by a real snapshot, and the snapshot is a
|
||||
// store a restore can put back: same rows, readable through the store.
|
||||
if got := taskTitles(t, out); len(got) != 1 || got[0] != "before" {
|
||||
t.Fatalf("snapshot tasks = %v, want the one seeded task", got)
|
||||
}
|
||||
if entries, err := filepath.Glob(out + ".tmp*"); err != nil || len(entries) != 0 {
|
||||
t.Errorf("backup left scratch files beside the snapshot: %v (%v)", entries, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The snapshot is the rollback state, so an interrupted backup must never
|
||||
// leave a truncated file where a good snapshot was.
|
||||
func TestRecoveryBackupRefusesAMissingDatabase(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
code, _, stderr := runRecoveryCmd(t, "backup", "-db", filepath.Join(dir, "absent.sqlite"), "-out", backupPath(t, dir))
|
||||
if code == 0 {
|
||||
t.Fatal("backup of a missing database succeeded")
|
||||
}
|
||||
if !strings.Contains(stderr, "absent.sqlite") {
|
||||
t.Errorf("refusal must name the database it could not read; stderr = %q", stderr)
|
||||
}
|
||||
}
|
||||
|
||||
// Restoring is the forward-migration escape hatch: the previous release
|
||||
// reads this file, so it must be exactly the snapshot, with the WAL that
|
||||
// belongs to the replaced database gone rather than replayed over it.
|
||||
func TestRecoveryRestorePutsTheSnapshotBackAndDropsItsWAL(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
db := filepath.Join(dir, "archie.db-tasks.sqlite")
|
||||
out := backupPath(t, dir)
|
||||
st := openTaskStore(t, db)
|
||||
seedTask(t, st, "before")
|
||||
backup(t, db, out)
|
||||
seedTask(t, st, "after")
|
||||
|
||||
code, stdout, stderr := runRecoveryCmd(t, "restore", "-db", db, "-from", out)
|
||||
if code != 0 {
|
||||
t.Fatalf("restore exited %d: %s", code, stderr)
|
||||
}
|
||||
if !strings.Contains(stdout, out) {
|
||||
t.Errorf("restore must report the snapshot it restored from; stdout = %q", stdout)
|
||||
}
|
||||
// Checked before anything reopens the database: opening a store sets WAL
|
||||
// mode, which creates these files, so the point is that the restore did not
|
||||
// leave the replaced database's log behind -- not that a later reader
|
||||
// cannot recreate its own.
|
||||
for _, suffix := range []string{"-wal", "-shm"} {
|
||||
if _, err := os.Stat(db + suffix); err == nil {
|
||||
t.Errorf("%s survived the restore: the replaced database's write-ahead log must not be replayed over the snapshot", db+suffix)
|
||||
}
|
||||
}
|
||||
if got := taskTitles(t, db); len(got) != 1 || got[0] != "before" {
|
||||
t.Fatalf("restored tasks = %v, want only the snapshot's task", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Overwriting a database a running State Store still owns leaves that process
|
||||
// writing to an unlinked inode while the store serves a file nobody owns --
|
||||
// the one way an offline command can destroy a task store.
|
||||
func TestRecoveryRestoreRefusesAStoreTheStateStoreIsRunning(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
db := filepath.Join(dir, "archie.db-tasks.sqlite")
|
||||
out := backupPath(t, dir)
|
||||
st := openTaskStore(t, db)
|
||||
seedTask(t, st, "before")
|
||||
backup(t, db, out)
|
||||
seedTask(t, st, "after")
|
||||
holdStoreOwnership(t, db)
|
||||
|
||||
code, _, stderr := runRecoveryCmd(t, "restore", "-db", db, "-from", out)
|
||||
if code == 0 {
|
||||
t.Fatal("restore against a live State Store succeeded")
|
||||
}
|
||||
if !strings.Contains(stderr, "owned by another process") {
|
||||
t.Errorf("refusal must say the database is owned by another process; stderr = %q", stderr)
|
||||
}
|
||||
if got := taskTitles(t, db); len(got) != 2 {
|
||||
t.Fatalf("refused restore changed the database: tasks = %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoveryRestoreRefusesAFileThatIsNotAStore(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
db := filepath.Join(dir, "archie.db-tasks.sqlite")
|
||||
seedTask(t, openTaskStore(t, db), "live")
|
||||
notAStore := filepath.Join(dir, "notes.txt")
|
||||
if err := os.WriteFile(notAStore, []byte("this is not a database"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
code, _, stderr := runRecoveryCmd(t, "restore", "-db", db, "-from", notAStore)
|
||||
if code == 0 {
|
||||
t.Fatal("restore from a file that is not a store succeeded")
|
||||
}
|
||||
if !strings.Contains(stderr, notAStore) {
|
||||
t.Errorf("refusal must name the snapshot it could not read; stderr = %q", stderr)
|
||||
}
|
||||
if got := taskTitles(t, db); len(got) != 1 {
|
||||
t.Fatalf("refused restore changed the database: tasks = %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoveryRestoreRefusesTheDatabaseAsItsOwnSnapshot(t *testing.T) {
|
||||
// Restoring a database from itself would delete the write-ahead log of the
|
||||
// file it is about to copy, so the one case that must never reach the swap
|
||||
// is refused up front.
|
||||
db := filepath.Join(t.TempDir(), "archie.db-tasks.sqlite")
|
||||
seedTask(t, openTaskStore(t, db), "live")
|
||||
|
||||
code, _, stderr := runRecoveryCmd(t, "restore", "-db", db, "-from", db)
|
||||
if code == 0 {
|
||||
t.Fatal("restore from the database itself succeeded")
|
||||
}
|
||||
if !strings.Contains(stderr, "is the database itself") {
|
||||
t.Errorf("refusal must say the snapshot is the database; stderr = %q", stderr)
|
||||
}
|
||||
if got := taskTitles(t, db); len(got) != 1 {
|
||||
t.Fatalf("refused restore changed the database: tasks = %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoveryValidateAcceptsAHealthyStore(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
db := filepath.Join(dir, "archie.db-tasks.sqlite")
|
||||
st := openTaskStore(t, db)
|
||||
seedTask(t, st, "healthy")
|
||||
seedResource(t, st, controlplane.ModelRoleAssignmentsKind, `{"implement":"anthropic/claude"}`, "seed")
|
||||
|
||||
code, stdout, stderr := runRecoveryCmd(t, "validate", "-db", db)
|
||||
if code != 0 {
|
||||
t.Fatalf("validate exited %d: %s", code, stderr)
|
||||
}
|
||||
if !strings.Contains(stdout, "schema version") {
|
||||
t.Errorf("validate must report the schema version it verified; stdout = %q", stdout)
|
||||
}
|
||||
}
|
||||
|
||||
// validate exists to answer "would archied start against this file". Each case
|
||||
// below is a state the daemon fails closed on, or a file it cannot read at all.
|
||||
func TestRecoveryValidateRefusesWhatTheDaemonRefuses(t *testing.T) {
|
||||
newStore := func(t *testing.T) string {
|
||||
t.Helper()
|
||||
db := filepath.Join(t.TempDir(), "archie.db-tasks.sqlite")
|
||||
openTaskStore(t, db)
|
||||
return db
|
||||
}
|
||||
|
||||
t.Run("forward_migrated_schema", func(t *testing.T) {
|
||||
// A database written by a newer release is the state a rollback has to
|
||||
// recognise before it hands the file to an older binary.
|
||||
db := newStore(t)
|
||||
if err := execRaw(t, db, `PRAGMA user_version = 99`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
code, _, stderr := runRecoveryCmd(t, "validate", "-db", db)
|
||||
if code == 0 {
|
||||
t.Fatal("validate accepted a forward-migrated database")
|
||||
}
|
||||
if !strings.Contains(stderr, "newer than supported version") {
|
||||
t.Errorf("refusal must name the schema version; stderr = %q", stderr)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("corrupt_file", func(t *testing.T) {
|
||||
// A real database header over pages that are not a database. The magic
|
||||
// is right, so this fails on the content rather than on the format, which
|
||||
// is the shape a half-written or truncated store has.
|
||||
db := filepath.Join(t.TempDir(), "archie.db-tasks.sqlite")
|
||||
corrupt := append([]byte("SQLite format 3\x00"), bytes.Repeat([]byte{0xff}, 4096)...)
|
||||
if err := os.WriteFile(db, corrupt, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
code, _, stderr := runRecoveryCmd(t, "validate", "-db", db)
|
||||
if code == 0 {
|
||||
t.Fatal("validate accepted a corrupt database")
|
||||
}
|
||||
if !strings.Contains(stderr, db) {
|
||||
t.Errorf("refusal must name the file; stderr = %q", stderr)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("not_a_database", func(t *testing.T) {
|
||||
// Pointing -db at the wrong file is the common operator mistake, and the
|
||||
// configured db_path is a different file from the task database the
|
||||
// State Store owns.
|
||||
db := filepath.Join(t.TempDir(), "archie.db")
|
||||
if err := os.WriteFile(db, []byte("db_path, not the task database"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
code, _, stderr := runRecoveryCmd(t, "validate", "-db", db)
|
||||
if code == 0 {
|
||||
t.Fatal("validate accepted a file that is not a database")
|
||||
}
|
||||
if !strings.Contains(stderr, db) {
|
||||
t.Errorf("refusal must name the file; stderr = %q", stderr)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("stored_resource_that_will_not_validate", func(t *testing.T) {
|
||||
// The stored settings value that stops archied starting: it can only
|
||||
// get here by bypassing the write path, which is exactly the state the
|
||||
// control plane fails closed on and the operator has no path back from.
|
||||
db := newStore(t)
|
||||
seedResource(t, openTaskStore(t, db), controlplane.ModelRoleAssignmentsKind, `{"implement":"claude"}`, "hand-edited")
|
||||
code, _, stderr := runRecoveryCmd(t, "validate", "-db", db)
|
||||
if code == 0 {
|
||||
t.Fatal("validate accepted a stored resource the write path would refuse")
|
||||
}
|
||||
if !strings.Contains(stderr, controlplane.ModelRoleAssignmentsKind) {
|
||||
t.Errorf("refusal must name the resource kind; stderr = %q", stderr)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("missing_database", func(t *testing.T) {
|
||||
db := newStore(t)
|
||||
if err := os.Remove(db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
code, _, stderr := runRecoveryCmd(t, "validate", "-db", db)
|
||||
if code == 0 {
|
||||
t.Fatal("validate accepted a database that does not exist")
|
||||
}
|
||||
if !strings.Contains(stderr, db) {
|
||||
t.Errorf("refusal must name the file; stderr = %q", stderr)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("store_without_the_control_plane_table", func(t *testing.T) {
|
||||
// A store written before the control plane existed holds no stored
|
||||
// settings. The serving process creates the table on its next start, so
|
||||
// refusing this file would send an operator to restore a snapshot for a
|
||||
// database that would have started perfectly well.
|
||||
db := newStore(t)
|
||||
if err := execRaw(t, db, `DROP TABLE resources`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
code, stdout, stderr := runRecoveryCmd(t, "validate", "-db", db)
|
||||
if code != 0 {
|
||||
t.Fatalf("validate exited %d: %s", code, stderr)
|
||||
}
|
||||
if !strings.Contains(stdout, "0 stored resources validate") {
|
||||
t.Errorf("validate must report that there was nothing stored to check; stdout = %q", stdout)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// rollback is the one operation that closes the documented gap: with the State
|
||||
// Store stopped, it replays the value an earlier revision recorded through the
|
||||
// ordinary replace, so no rollback RPC is needed and the rollback itself is
|
||||
// audited as one more revision.
|
||||
func TestRecoveryRollbackReplaysTheRevisionThroughReplace(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
db := filepath.Join(dir, "archie.db-tasks.sqlite")
|
||||
st := openTaskStore(t, db)
|
||||
seedResource(t, st, controlplane.ModelRoleAssignmentsKind, `{"implement":"anthropic/claude"}`, "first")
|
||||
seedResource(t, st, controlplane.ModelRoleAssignmentsKind, `{"implement":"openai/gpt"}`, "second")
|
||||
|
||||
code, stdout, stderr := runRecoveryCmd(t, "rollback", "-db", db, "-kind", controlplane.ModelRoleAssignmentsKind)
|
||||
if code != 0 {
|
||||
t.Fatalf("rollback exited %d: %s", code, stderr)
|
||||
}
|
||||
if !strings.Contains(stdout, controlplane.ModelRoleAssignmentsKind) {
|
||||
t.Errorf("rollback must name the resource it replayed; stdout = %q", stdout)
|
||||
}
|
||||
resource, err := st.Resource(t.Context(), controlplane.ModelRoleAssignmentsKind)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(resource.Value) != `{"implement":"anthropic/claude"}` {
|
||||
t.Errorf("resource value = %s, want the first revision's value", resource.Value)
|
||||
}
|
||||
if resource.Version != 3 {
|
||||
t.Errorf("resource version = %d, want a new revision (3) rather than a rewrite of the old one", resource.Version)
|
||||
}
|
||||
history, err := st.ResourceHistory(t.Context(), controlplane.ModelRoleAssignmentsKind, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(history) != 3 {
|
||||
t.Fatalf("history has %d revisions, want the rollback recorded as the third", len(history))
|
||||
}
|
||||
if history[0].Source != archied.OfflineRollbackSource {
|
||||
t.Errorf("newest revision source = %q, want %q so the rollback is auditable", history[0].Source, archied.OfflineRollbackSource)
|
||||
}
|
||||
}
|
||||
|
||||
// The revision to restore may be named, which is how an operator reaches past
|
||||
// the most recent change without replaying it first.
|
||||
func TestRecoveryRollbackRestoresANamedRevision(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
db := filepath.Join(dir, "archie.db-tasks.sqlite")
|
||||
st := openTaskStore(t, db)
|
||||
seedResource(t, st, controlplane.ModelRoleAssignmentsKind, `{"implement":"anthropic/claude"}`, "first")
|
||||
seedResource(t, st, controlplane.ModelRoleAssignmentsKind, `{"implement":"openai/gpt"}`, "second")
|
||||
seedResource(t, st, controlplane.ModelRoleAssignmentsKind, `{"implement":"google/gemini"}`, "third")
|
||||
|
||||
code, _, stderr := runRecoveryCmd(t, "rollback", "-db", db, "-kind", controlplane.ModelRoleAssignmentsKind, "-revision", "1")
|
||||
if code != 0 {
|
||||
t.Fatalf("rollback exited %d: %s", code, stderr)
|
||||
}
|
||||
resource, err := st.Resource(t.Context(), controlplane.ModelRoleAssignmentsKind)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(resource.Value) != `{"implement":"anthropic/claude"}` {
|
||||
t.Errorf("resource value = %s, want the value recorded at revision 1", resource.Value)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoveryRollbackRefusesWhatItCannotReplay(t *testing.T) {
|
||||
newStore := func(t *testing.T, revisions int) string {
|
||||
t.Helper()
|
||||
db := filepath.Join(t.TempDir(), "archie.db-tasks.sqlite")
|
||||
st := openTaskStore(t, db)
|
||||
for i := range revisions {
|
||||
value := `{"implement":"anthropic/claude"}`
|
||||
if i > 0 {
|
||||
value = `{"implement":"openai/gpt"}`
|
||||
}
|
||||
seedResource(t, st, controlplane.ModelRoleAssignmentsKind, value, "seed")
|
||||
}
|
||||
return db
|
||||
}
|
||||
|
||||
t.Run("unknown_kind", func(t *testing.T) {
|
||||
db := newStore(t, 2)
|
||||
code, _, stderr := runRecoveryCmd(t, "rollback", "-db", db, "-kind", "not-a-resource")
|
||||
if code == 0 {
|
||||
t.Fatal("rollback of an unknown resource kind succeeded")
|
||||
}
|
||||
if !strings.Contains(stderr, "not-a-resource") {
|
||||
t.Errorf("refusal must name the kind; stderr = %q", stderr)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("no_earlier_revision", func(t *testing.T) {
|
||||
db := newStore(t, 1)
|
||||
code, _, stderr := runRecoveryCmd(t, "rollback", "-db", db, "-kind", controlplane.ModelRoleAssignmentsKind)
|
||||
if code == 0 {
|
||||
t.Fatal("rollback with no earlier revision succeeded")
|
||||
}
|
||||
if !strings.Contains(stderr, "no earlier revision") {
|
||||
t.Errorf("refusal must say there is nothing to roll back to; stderr = %q", stderr)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("current_revision", func(t *testing.T) {
|
||||
db := newStore(t, 2)
|
||||
code, _, stderr := runRecoveryCmd(t, "rollback", "-db", db, "-kind", controlplane.ModelRoleAssignmentsKind, "-revision", "2")
|
||||
if code == 0 {
|
||||
t.Fatal("rollback of the current revision succeeded")
|
||||
}
|
||||
if !strings.Contains(stderr, "is the current version") {
|
||||
t.Errorf("refusal must say the revision is already the current value; stderr = %q", stderr)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("live_state_store", func(t *testing.T) {
|
||||
db := newStore(t, 2)
|
||||
holdStoreOwnership(t, db)
|
||||
code, _, stderr := runRecoveryCmd(t, "rollback", "-db", db, "-kind", controlplane.ModelRoleAssignmentsKind)
|
||||
if code == 0 {
|
||||
t.Fatal("rollback against a live State Store succeeded")
|
||||
}
|
||||
if !strings.Contains(stderr, "owned by another process") {
|
||||
t.Errorf("refusal must say the database is owned by another process; stderr = %q", stderr)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// A positional argument that is not a subcommand used to be ignored, leaving
|
||||
// an operator who mistyped a command watching a server start instead.
|
||||
func TestRecoveryRejectsAnUnknownSubcommand(t *testing.T) {
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := runRecovery([]string{"backp", "-db", "x"}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("an unknown subcommand exited 0")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "backp") {
|
||||
t.Errorf("refusal must name the unknown subcommand; stderr = %q", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
// seedResource writes one revision of a control-plane resource through the
|
||||
// store, the way the control plane's replace does. The request ID is derived
|
||||
// from the label and the revision it produces: the store deduplicates a
|
||||
// repeated request ID, so a fixed one would make the second write of a series
|
||||
// a silent no-op.
|
||||
func seedResource(t *testing.T, st *store.Store, kind, value, label string) {
|
||||
t.Helper()
|
||||
current, err := st.Resource(t.Context(), kind)
|
||||
expected := int64(0)
|
||||
if err == nil {
|
||||
expected = current.Version
|
||||
}
|
||||
var document any
|
||||
if err := json.Unmarshal([]byte(value), &document); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
encoded, err := json.Marshal(document)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := st.PutResource(t.Context(), store.ResourceWrite{
|
||||
Kind: kind, Value: encoded, Actor: label, Source: "test",
|
||||
RequestID: fmt.Sprintf("%s-%d", label, expected+1), ExpectedVersion: expected,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed resource %s: %v", kind, err)
|
||||
}
|
||||
}
|
||||
|
||||
// execRaw runs one statement against the database file the way an operator
|
||||
// editing a broken store would, bypassing every check the store applies.
|
||||
func execRaw(t *testing.T, path, statement string) error {
|
||||
t.Helper()
|
||||
db, err := sql.Open("sqlite", path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = db.Close() }()
|
||||
_, err = db.ExecContext(t.Context(), statement)
|
||||
return err
|
||||
}
|
||||
@@ -120,21 +120,40 @@ file values would restore the second source of truth the control plane exists to
|
||||
remove. The cost is that a bad stored value stops the daemon starting, and the
|
||||
value can only be corrected through the daemon's own API.
|
||||
|
||||
### Recovery: not yet built
|
||||
### Recovery: offline commands on the State Store
|
||||
|
||||
There is no escape hatch. Nothing skips the control plane at boot, and the only
|
||||
writer for a resource is `ControlPlaneService.Command` on a running State Store.
|
||||
An operator whose stored settings will not validate has no supported path back
|
||||
short of editing `archie.db` by hand.
|
||||
Nothing skips the control plane at boot, and the only writer for a resource is
|
||||
`ControlPlaneService.Command` on a running State Store -- which is exactly what
|
||||
the daemon fails closed without. `archie-state-store` therefore carries the
|
||||
offline recovery commands (`archie-core-6zw0`). They work on the task database
|
||||
file directly -- the configured `db_path` with `-tasks.sqlite` appended, not
|
||||
`db_path` itself -- and need neither `archied` nor the Web UI.
|
||||
|
||||
`archie-core-6zw0` is the missing piece: offline commands that back up, restore,
|
||||
validate and roll back the database without `archied` or the Web UI. Until it
|
||||
lands, treat a settings change that requires a restart as a change that can
|
||||
prevent one, and take a copy of `archie.db` first.
|
||||
| Command | What it does | State Store |
|
||||
| --- | --- | --- |
|
||||
| `backup -db F -out F` | writes a transactionally consistent snapshot (`VACUUM INTO`), verified as a usable store before it is published | may be running |
|
||||
| `restore -db F -from F` | replaces the database with a snapshot and drops the replaced file's `-wal`/`-shm` | must be stopped |
|
||||
| `validate -db F` | checks the file the way boot meets it: readable, not corrupt, schema no newer than the binary, and every stored resource decoding under the definition that owns it | must be stopped |
|
||||
| `rollback -db F -kind K [-revision N]` | replays an earlier revision of a stored resource through the ordinary replace, recording the rollback as a new revision | must be stopped |
|
||||
|
||||
Resource history is the in-band remedy and does exist: every write records its
|
||||
prior value, so an edit made through the dashboard or a channel can be restored
|
||||
by replaying an earlier revision while the State Store is up.
|
||||
No rollback RPC exists and none is needed: a revision carries its own value, so
|
||||
restoring one is the same replace the dashboard and the channels already use.
|
||||
The offline command is that operation with the store stopped -- the one state in
|
||||
which the daemon's own API cannot serve it.
|
||||
|
||||
"Must be stopped" is checked rather than assumed. The serving process takes an
|
||||
exclusive flock on `<db>.lock` for its whole life, and `restore` and `rollback`
|
||||
refuse with `task database is owned by another process` while another process
|
||||
holds it. `backup` is the deliberate exception: the update installer runs inside
|
||||
the process it is replacing and cannot stop the State Store, so the snapshot
|
||||
that makes a failed update reversible is taken against a serving store -- and
|
||||
`scripts/archie-update-install` takes it through this subcommand rather than
|
||||
through its own `sqlite3` shell, so the installer and an operator run one
|
||||
implementation.
|
||||
|
||||
`rollback` is the surgical route back from a bad stored value; `restore` is the
|
||||
blunt one, and is also the way back from a schema migration a release cannot
|
||||
grow out of.
|
||||
|
||||
## Gateway restart: two constraints learned the hard way
|
||||
|
||||
|
||||
@@ -215,7 +215,25 @@ func (b *boot) openStateStore(ctx context.Context) error {
|
||||
log.Error("configure bindings cipher", "err", err)
|
||||
return err
|
||||
}
|
||||
st, err := openProductionTaskStore(ctx, taskDBPath(cfg.DBPath), store.WithBindingCipher(bindingCipher))
|
||||
// Claim the database before opening it. This process is the file's owner
|
||||
// for as long as it lives, and the claim is what an offline recovery
|
||||
// command checks before it rewrites the file: without it, "the State Store
|
||||
// is stopped" is a rule the operator is trusted to have followed rather
|
||||
// than a fact the command can verify. A second State Store on the same
|
||||
// database fails here, which is the single-owner invariant stated instead
|
||||
// of merely documented.
|
||||
path := taskDBPath(cfg.DBPath)
|
||||
ownership, err := store.AcquireOwnership(path)
|
||||
if err != nil {
|
||||
log.Error("claim state store ownership", "err", err)
|
||||
return err
|
||||
}
|
||||
b.addCleanup(func() {
|
||||
if err := ownership.Release(); err != nil {
|
||||
log.Error("release state store ownership", "err", err)
|
||||
}
|
||||
})
|
||||
st, err := openProductionTaskStore(ctx, path, store.WithBindingCipher(bindingCipher))
|
||||
if err != nil {
|
||||
log.Error("open state store", "err", err)
|
||||
return err
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
// state_store_recovery.go composes the offline recovery subcommands of the
|
||||
// standalone archie-state-store process. They exist because the control plane
|
||||
// fails closed: a stored setting that will not validate stops archied
|
||||
// starting, and the in-band remedy -- replaying an earlier revision while the
|
||||
// State Store is up -- is unavailable in exactly the case that needs it
|
||||
// (docs/architecture/safe-change-and-recovery.md).
|
||||
//
|
||||
// Every operation works on the task database file directly, with the State
|
||||
// Store stopped, and the ones that write refuse when another process owns the
|
||||
// file. Backup is the exception and deliberately so: the update installer
|
||||
// cannot stop the process that runs it, so the snapshot that makes a failed
|
||||
// update reversible is taken against a serving store through SQLite's own
|
||||
// consistent copy.
|
||||
package archied
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/app/controlplane"
|
||||
pb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1"
|
||||
"github.com/samcharles93/archie-core/internal/store"
|
||||
)
|
||||
|
||||
// The audit identity every offline rollback records. A rollback is one more
|
||||
// revision of the resource, written through the ordinary replace, so the
|
||||
// operator can see who went back and when -- and can go forward again the same
|
||||
// way, with the State Store up or down.
|
||||
const (
|
||||
OfflineRollbackSource = "archie-state-store rollback"
|
||||
OfflineRollbackActor = "operator:offline-recovery"
|
||||
)
|
||||
|
||||
// Recovery operations, as named on the command line.
|
||||
const (
|
||||
RecoveryBackup = "backup"
|
||||
RecoveryRestore = "restore"
|
||||
RecoveryValidate = "validate"
|
||||
RecoveryRollback = "rollback"
|
||||
)
|
||||
|
||||
// StateStoreRecoveryOptions are the process inputs for one offline recovery
|
||||
// operation.
|
||||
type StateStoreRecoveryOptions struct {
|
||||
// Operation is one of backup, restore, validate, rollback.
|
||||
Operation string
|
||||
// DB is the task database file the State Store owns: the configured
|
||||
// db_path with "-tasks.sqlite" appended, not the configured path itself.
|
||||
DB string
|
||||
// Out is the snapshot backup writes.
|
||||
Out string
|
||||
// From is the snapshot restore reads.
|
||||
From string
|
||||
// Kind is the control-plane resource kind rollback replays.
|
||||
Kind string
|
||||
// Revision is the revision rollback replays. Zero means the newest
|
||||
// revision older than the one the resource carries now.
|
||||
Revision int64
|
||||
}
|
||||
|
||||
// RunStateStoreRecovery performs one offline operation on the task database and
|
||||
// returns the line the command reports on stdout.
|
||||
func RunStateStoreRecovery(ctx context.Context, options StateStoreRecoveryOptions) (string, error) {
|
||||
switch options.Operation {
|
||||
case RecoveryBackup:
|
||||
if options.DB == "" || options.Out == "" {
|
||||
return "", errors.New("backup requires -db and -out")
|
||||
}
|
||||
if err := store.Backup(ctx, options.DB, options.Out); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("backed up %s to %s", options.DB, options.Out), nil
|
||||
case RecoveryRestore:
|
||||
if options.DB == "" || options.From == "" {
|
||||
return "", errors.New("restore requires -db and -from")
|
||||
}
|
||||
if err := store.Restore(ctx, options.DB, options.From); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("restored %s from %s; start the State Store again", options.DB, options.From), nil
|
||||
case RecoveryValidate:
|
||||
if options.DB == "" {
|
||||
return "", errors.New("validate requires -db")
|
||||
}
|
||||
return validateStore(ctx, options.DB)
|
||||
case RecoveryRollback:
|
||||
if options.DB == "" || options.Kind == "" {
|
||||
return "", errors.New("rollback requires -db and -kind")
|
||||
}
|
||||
return rollbackResource(ctx, options)
|
||||
default:
|
||||
return "", fmt.Errorf("unknown recovery command %q", options.Operation)
|
||||
}
|
||||
}
|
||||
|
||||
// validateStore answers "would archied start against this file". The file-level
|
||||
// checks come first, so the store is only read once its schema is one this
|
||||
// binary understands; the resource check then decodes every stored value with
|
||||
// the definition that owns it, which is the same validation a write applies.
|
||||
func validateStore(ctx context.Context, path string) (string, error) {
|
||||
version, err := store.ValidateFile(ctx, path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
st, err := store.OpenReadOnly(ctx, path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer func() { _ = st.Close() }()
|
||||
// The resources table is only absent from a store written before the control
|
||||
// plane existed. That store holds no settings to validate, and the serving
|
||||
// process creates the table on its next start -- refusing it here would send
|
||||
// an operator to restore a snapshot for nothing.
|
||||
checked := 0
|
||||
stored, err := st.StoresResources(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if stored {
|
||||
checked, err = controlplane.NewServer(st).ValidateStored(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
return fmt.Sprintf("%s is a valid store at schema version %d; %d stored resources validate", path, version, checked), nil
|
||||
}
|
||||
|
||||
// rollbackResource is the one operation the Web UI cannot be asked to perform
|
||||
// here: the daemon fails closed on the bad value, so the store that would serve
|
||||
// the rollback is the store that will not start. It replays the revision's own
|
||||
// value through the ordinary replace, so no rollback RPC exists or is needed,
|
||||
// and the rollback is audited as one more revision.
|
||||
func rollbackResource(ctx context.Context, options StateStoreRecoveryOptions) (string, error) {
|
||||
// Before the lock and before the open: opening a store creates the file,
|
||||
// and a rollback pointed at a mistyped path must not leave an empty
|
||||
// database where the operator expected theirs.
|
||||
if err := store.RequireDatabase(options.DB); err != nil {
|
||||
return "", err
|
||||
}
|
||||
ownership, err := store.AcquireOwnership(options.DB)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer func() { _ = ownership.Release() }()
|
||||
|
||||
st, err := store.Open(ctx, options.DB)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer func() { _ = st.Close() }()
|
||||
|
||||
server := controlplane.NewServer(st)
|
||||
if !server.Owns(options.Kind) {
|
||||
return "", fmt.Errorf("unknown resource kind %q", options.Kind)
|
||||
}
|
||||
current, err := st.Resource(ctx, options.Kind)
|
||||
if errors.Is(err, store.ErrResourceNotFound) {
|
||||
return "", fmt.Errorf("%s has no stored value to roll back", options.Kind)
|
||||
}
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read %s: %w", options.Kind, err)
|
||||
}
|
||||
revision, err := revisionToReplay(ctx, st, options.Kind, current.Version, options.Revision)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
replaced, err := server.Command(ctx, &pb.CommandRequest{
|
||||
Kind: options.Kind,
|
||||
Command: "replace",
|
||||
ValueJson: revision.Value,
|
||||
ExpectedVersion: current.Version,
|
||||
Actor: OfflineRollbackActor,
|
||||
Source: OfflineRollbackSource,
|
||||
RequestId: fmt.Sprintf("recovery-rollback-%s-%d", options.Kind, time.Now().UTC().UnixNano()),
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("replay %s at revision %d: %w", options.Kind, revision.Version, err)
|
||||
}
|
||||
return fmt.Sprintf("rolled back %s from version %d to the value recorded at version %d; the store now holds version %d",
|
||||
options.Kind, current.Version, revision.Version, replaced.GetResource().GetVersion()), nil
|
||||
}
|
||||
|
||||
// revisionToReplay picks the value to put back: the revision the operator
|
||||
// named, or the newest one older than the value the resource carries now.
|
||||
func revisionToReplay(ctx context.Context, st *store.Store, kind string, current, requested int64) (store.Resource, error) {
|
||||
history, err := st.ResourceHistory(ctx, kind, 0)
|
||||
if err != nil {
|
||||
return store.Resource{}, fmt.Errorf("read %s history: %w", kind, err)
|
||||
}
|
||||
if requested > 0 {
|
||||
for _, revision := range history {
|
||||
if revision.Version != requested {
|
||||
continue
|
||||
}
|
||||
if revision.Version == current {
|
||||
return store.Resource{}, fmt.Errorf("%s revision %d is the current version; there is nothing to roll back to", kind, requested)
|
||||
}
|
||||
return revision, nil
|
||||
}
|
||||
return store.Resource{}, fmt.Errorf("%s has no revision %d", kind, requested)
|
||||
}
|
||||
for _, revision := range history {
|
||||
if revision.Version < current {
|
||||
return revision, nil
|
||||
}
|
||||
}
|
||||
return store.Resource{}, fmt.Errorf("%s has no earlier revision to roll back to", kind)
|
||||
}
|
||||
@@ -89,6 +89,42 @@ func (s *Server) ImportConfig(ctx context.Context, cfg config.Config) (map[strin
|
||||
return versions, nil
|
||||
}
|
||||
|
||||
// Owns reports whether kind is a resource the control plane owns, so a caller
|
||||
// acting on a stored value can refuse a kind nobody defines before it reaches
|
||||
// the store.
|
||||
func (s *Server) Owns(kind string) bool {
|
||||
_, ok := s.definitions[kind]
|
||||
return ok
|
||||
}
|
||||
|
||||
// ValidateStored decodes every stored resource with the definition that owns
|
||||
// it, using the same Decode the write path applies, and reports how many
|
||||
// resources it checked. Errors name the kind and the revision it was stored at,
|
||||
// which is what an operator needs to roll the value back.
|
||||
//
|
||||
// This is how an offline validate reproduces the refusal that stops archied
|
||||
// starting: the check is the writer's own, so it cannot drift from what a
|
||||
// replace would accept, and a value can only be stored unvalidated by bypassing
|
||||
// the write path entirely.
|
||||
func (s *Server) ValidateStored(ctx context.Context) (int, error) {
|
||||
checked := 0
|
||||
var failures []error
|
||||
for _, definition := range s.ordered {
|
||||
resource, err := s.store.Resource(ctx, definition.Kind)
|
||||
if errors.Is(err, store.ErrResourceNotFound) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return checked, fmt.Errorf("read %s: %w", definition.Kind, err)
|
||||
}
|
||||
checked++
|
||||
if _, err := definition.Decode(resource.Value); err != nil {
|
||||
failures = append(failures, fmt.Errorf("%s at version %d: %w", definition.Kind, resource.Version, err))
|
||||
}
|
||||
}
|
||||
return checked, errors.Join(failures...)
|
||||
}
|
||||
|
||||
func validateAs[T any](input []byte, validate func(T) error) error {
|
||||
var value T
|
||||
decoder := json.NewDecoder(bytes.NewReader(input))
|
||||
|
||||
@@ -382,6 +382,13 @@ esac
|
||||
`)
|
||||
writeFakeCommand(t, fakeDir, "go", `
|
||||
printf '%s\n' "go $*" >> "$ARCHIE_TEST_CALLS"
|
||||
# The pre-flight task-database backup runs through the release's own
|
||||
# archie-state-store subcommand, so a release whose recovery path cannot run
|
||||
# must refuse the update.
|
||||
if [ "$1" = "run" ]; then
|
||||
[ "${ARCHIE_TEST_GO_FAILS_RUN:-}" = "1" ] && exit 1
|
||||
exit 0
|
||||
fi
|
||||
out=""
|
||||
while [ "$#" -gt 0 ]; do
|
||||
if [ "$1" = "-o" ]; then out="$2"; break; fi
|
||||
@@ -745,6 +752,10 @@ func TestUpdateWatchdogCyclesEveryUnitInDependencyOrder(t *testing.T) {
|
||||
// value up produces an empty backup, and a rollback then restores that empty
|
||||
// file over the database and deletes its -wal/-shm. A rollback that destroys
|
||||
// the task store is worse than no rollback.
|
||||
//
|
||||
// The snapshot is taken by the release's own recovery subcommand rather than
|
||||
// by hand-rolled shell, so the installer and an operator run the same code and
|
||||
// the two cannot drift apart.
|
||||
func TestUpdateInstallBacksUpTheSiblingTaskDatabase(t *testing.T) {
|
||||
work := t.TempDir()
|
||||
configured := filepath.Join(work, "archie.db")
|
||||
@@ -764,7 +775,8 @@ func TestUpdateInstallBacksUpTheSiblingTaskDatabase(t *testing.T) {
|
||||
"ARCHIE_TASK_DB_BACKUP": backup,
|
||||
}, "db_path = \""+configured+"\"\n\n[containers]\nimage = 'registry.example/archie-agent:stable'\n", false)
|
||||
|
||||
assertCallContains(t, calls, "sqlite3", realStore, ".backup")
|
||||
assertCallContains(t, calls, "go run ./cmd/archie-state-store backup", "-db "+realStore, "-out "+backup)
|
||||
assertCallAbsent(t, calls, "sqlite3")
|
||||
assertCallContains(t, calls, "systemd-run", "--setenv=ARCHIE_TASK_DB_PATH="+realStore)
|
||||
// The zero-byte placeholder is not the store, and must never be the thing
|
||||
// handed to the watchdog as ARCHIE_TASK_DB_PATH.
|
||||
@@ -793,9 +805,9 @@ func TestUpdateInstallIgnoresDbPathFromOtherSections(t *testing.T) {
|
||||
"ARCHIE_TASK_DB_BACKUP": backup,
|
||||
}, "db_path = \""+configured+"\"\n\n[indexing]\ndb_path = \""+indexStore+"\"\n\n[containers]\nimage = 'registry.example/archie-agent:stable'\n", false)
|
||||
|
||||
assertCallContains(t, calls, "sqlite3", realStore)
|
||||
assertCallContains(t, calls, "go run ./cmd/archie-state-store backup", "-db "+realStore)
|
||||
for _, call := range calls {
|
||||
if strings.Contains(call, "sqlite3") && strings.Contains(call, indexStore) {
|
||||
if strings.Contains(call, "go run ./cmd/archie-state-store backup") && strings.Contains(call, indexStore) {
|
||||
t.Errorf("backed up the indexing database %s: %q", indexStore, call)
|
||||
}
|
||||
}
|
||||
@@ -815,7 +827,7 @@ func TestUpdateInstallRefusesWhenTaskDatabaseCannotBeBackedUp(t *testing.T) {
|
||||
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0",
|
||||
"ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0",
|
||||
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0",
|
||||
"ARCHIE_TEST_SQLITE3_FAILS": "1",
|
||||
"ARCHIE_TEST_GO_FAILS_RUN": "1",
|
||||
}, "db_path = \""+configured+"\"\n\n[containers]\nimage = 'registry.example/archie-agent:stable'\n", true)
|
||||
|
||||
if !strings.Contains(output, "task database") {
|
||||
|
||||
@@ -0,0 +1,253 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// This file is the offline half of the store's contract with its owner: the
|
||||
// operations the State Store process cannot perform on itself, because they
|
||||
// read or replace the very file it has open. They are what
|
||||
// docs/prds/runtime-control-plane.md ("Bootstrap, migration, and recovery")
|
||||
// asks for -- back up, restore and validate the database without archied or
|
||||
// the Web UI -- and every one of them works on the file directly.
|
||||
//
|
||||
// The read-only operations never take the ownership lock, because they are
|
||||
// what an operator runs against a serving store. The one that replaces the
|
||||
// file does, because a rewrite under a running server is the only way an
|
||||
// offline command can destroy a task store.
|
||||
|
||||
// OpenReadOnly opens an existing store for reading only: no schema is created
|
||||
// and no migration runs. That matters for recovery -- the snapshot an update
|
||||
// takes has to stay exactly what the release that wrote it expects, and
|
||||
// migrating a copy during verification would hand a rollback a schema its
|
||||
// binary refuses.
|
||||
func OpenReadOnly(ctx context.Context, path string) (*Store, error) {
|
||||
if err := RequireDatabase(path); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
db, err := sql.Open("sqlite", readonlyDSN(path))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
db.SetMaxOpenConns(1)
|
||||
// sql.Open is lazy: the first statement is where a file that is not a
|
||||
// database, a missing WAL sidecar or an unreadable path surfaces.
|
||||
var tables int
|
||||
if err := db.QueryRowContext(ctx, `SELECT count(*) FROM sqlite_master`).Scan(&tables); err != nil {
|
||||
return nil, errors.Join(fmt.Errorf("read task database %s: %w", path, err), db.Close())
|
||||
}
|
||||
return &Store{db: db}, nil
|
||||
}
|
||||
|
||||
// readonlyDSN reads the store without the write-ahead log pragma the serving
|
||||
// process sets: a reader has no business changing the database's journal mode,
|
||||
// and the read-only flag makes an accidental write an error rather than a
|
||||
// silent modification of a file someone else owns.
|
||||
func readonlyDSN(path string) string {
|
||||
separator := "?"
|
||||
if strings.HasSuffix(path, "?") || strings.HasSuffix(path, "&") {
|
||||
separator = ""
|
||||
} else if strings.Contains(path, "?") {
|
||||
separator = "&"
|
||||
}
|
||||
return path + separator + "mode=ro&_pragma=busy_timeout(5000)"
|
||||
}
|
||||
|
||||
// StoresResources reports whether the store's file carries the control-plane
|
||||
// resources table. A store written before the control plane existed has none
|
||||
// and holds no stored settings to validate; the serving process creates the
|
||||
// table on its next start, so an offline check must tell that apart from
|
||||
// resources it cannot read.
|
||||
func (s *Store) StoresResources(ctx context.Context) (bool, error) {
|
||||
var tables int
|
||||
if err := s.db.QueryRowContext(ctx, `SELECT count(*) FROM sqlite_master WHERE type='table' AND name='resources'`).Scan(&tables); err != nil {
|
||||
return false, fmt.Errorf("inspect store schema: %w", err)
|
||||
}
|
||||
return tables > 0, nil
|
||||
}
|
||||
|
||||
// RequireDatabase reports whether path is an existing store file. A writer
|
||||
// that must not create the file it fails to find asks this first: a recovery
|
||||
// command pointed at a mistyped path has to refuse, not leave a fresh empty
|
||||
// database where an operator expected theirs.
|
||||
func RequireDatabase(path string) error {
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
return fmt.Errorf("task database %s: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ValidateFile checks an existing store the way the serving process meets it
|
||||
// on startup and reports the schema version it carries: the file is a SQLite
|
||||
// database this binary can read, it is not corrupt, and its schema is not
|
||||
// newer than this binary supports.
|
||||
func ValidateFile(ctx context.Context, path string) (int, error) {
|
||||
st, err := OpenReadOnly(ctx, path)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer func() { _ = st.Close() }()
|
||||
return st.check(ctx)
|
||||
}
|
||||
|
||||
// check runs the file-level checks. integrity_check returns one row per
|
||||
// problem, or the single row "ok"; reading the first row is therefore the
|
||||
// verdict rather than a sample.
|
||||
func (s *Store) check(ctx context.Context) (int, error) {
|
||||
var result string
|
||||
if err := s.db.QueryRowContext(ctx, `PRAGMA integrity_check`).Scan(&result); err != nil {
|
||||
return 0, fmt.Errorf("integrity check: %w", err)
|
||||
}
|
||||
if result != "ok" {
|
||||
return 0, fmt.Errorf("integrity check: %s", result)
|
||||
}
|
||||
var version int
|
||||
if err := s.db.QueryRowContext(ctx, `PRAGMA user_version`).Scan(&version); err != nil {
|
||||
return 0, fmt.Errorf("read schema version: %w", err)
|
||||
}
|
||||
if version > taskSchemaVersion {
|
||||
return 0, fmt.Errorf("database schema version %d is newer than supported version %d", version, taskSchemaVersion)
|
||||
}
|
||||
return version, nil
|
||||
}
|
||||
|
||||
// Backup writes a transactionally consistent snapshot of database to snapshot,
|
||||
// using SQLite's own copy, so a store that is being written to cannot produce
|
||||
// a torn file.
|
||||
//
|
||||
// It deliberately does not take the ownership lock. The update installer runs
|
||||
// inside the process it is updating and cannot stop the State Store, so the
|
||||
// snapshot that makes a failed update reversible is taken against a serving
|
||||
// store; a backup that insisted on an exclusively owned file would break the
|
||||
// one caller that exists. The snapshot is written beside the destination and
|
||||
// renamed into place, so an interrupted backup never replaces a snapshot that
|
||||
// was good with one that is not.
|
||||
func Backup(ctx context.Context, database, snapshot string) error {
|
||||
if err := RequireDatabase(database); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(snapshot), 0o755); err != nil {
|
||||
return fmt.Errorf("create snapshot directory: %w", err)
|
||||
}
|
||||
db, err := sql.Open("sqlite", sqliteDSN(database))
|
||||
if err != nil {
|
||||
return fmt.Errorf("open task database %s: %w", database, err)
|
||||
}
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
staging := snapshot + ".tmp"
|
||||
if err := os.Remove(staging); err != nil && !os.IsNotExist(err) {
|
||||
return fmt.Errorf("clear staging snapshot %s: %w", staging, err)
|
||||
}
|
||||
if _, err := db.ExecContext(ctx, `VACUUM INTO ?`, staging); err != nil {
|
||||
return errors.Join(fmt.Errorf("snapshot %s: %w", database, err), os.Remove(staging))
|
||||
}
|
||||
// A snapshot that is not itself a usable store is worse than no snapshot:
|
||||
// the update path treats its existence as proof that a rollback is
|
||||
// possible.
|
||||
if _, err := ValidateFile(ctx, staging); err != nil {
|
||||
return errors.Join(fmt.Errorf("snapshot is not a usable store: %w", err), os.Remove(staging))
|
||||
}
|
||||
if err := os.Rename(staging, snapshot); err != nil {
|
||||
return errors.Join(fmt.Errorf("publish snapshot %s: %w", snapshot, err), os.Remove(staging))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Restore replaces database with the snapshot at snapshot. The snapshot is
|
||||
// verified before anything is destroyed, so a file that is not a usable store
|
||||
// cannot take the database down with it, and the database is replaced by a
|
||||
// rename, so a failure leaves either the old file or the new one rather than a
|
||||
// mixture.
|
||||
//
|
||||
// It holds the ownership lock for its duration: rewriting a database a running
|
||||
// State Store still has open would leave that process writing to an unlinked
|
||||
// inode while the store served a file nobody owns.
|
||||
func Restore(ctx context.Context, database, snapshot string) error {
|
||||
if _, err := os.Stat(filepath.Dir(database)); err != nil {
|
||||
return fmt.Errorf("restore destination %s: %w", database, err)
|
||||
}
|
||||
if same, err := sameFile(database, snapshot); err != nil {
|
||||
return err
|
||||
} else if same {
|
||||
return fmt.Errorf("restore snapshot %s is the database itself", snapshot)
|
||||
}
|
||||
ownership, err := AcquireOwnership(database)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = ownership.Release() }()
|
||||
|
||||
if _, err := ValidateFile(ctx, snapshot); err != nil {
|
||||
return fmt.Errorf("snapshot is not a usable store: %w", err)
|
||||
}
|
||||
staging := database + ".restore.tmp"
|
||||
if err := copyFile(snapshot, staging); err != nil {
|
||||
return errors.Join(err, os.Remove(staging))
|
||||
} // The replaced database's write-ahead log belongs to the file that is
|
||||
// going away. It goes first: between the two calls the database is merely
|
||||
// incomplete, whereas a WAL left beside the file it does not describe is
|
||||
// the state SQLite cannot recover from.
|
||||
for _, sidecar := range []string{"-wal", "-shm"} {
|
||||
if err := os.Remove(database + sidecar); err != nil && !os.IsNotExist(err) {
|
||||
return errors.Join(fmt.Errorf("remove %s%s: %w", database, sidecar, err), os.Remove(staging))
|
||||
}
|
||||
}
|
||||
if err := os.Rename(staging, database); err != nil {
|
||||
return errors.Join(fmt.Errorf("publish restored database %s: %w", database, err), os.Remove(staging))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sameFile(a, b string) (bool, error) {
|
||||
first, err := os.Stat(a)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
second, err := os.Stat(b)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return os.SameFile(first, second), nil
|
||||
}
|
||||
|
||||
// copyFile writes source over a new file at path, flushed to disk before it is
|
||||
// returned: the rename that publishes it must not reach the directory entries
|
||||
// ahead of the bytes they point at. The snapshot's own permissions are kept, so
|
||||
// a restore does not silently change who may read the operator's database.
|
||||
func copyFile(source, path string) (retErr error) {
|
||||
info, err := os.Stat(source)
|
||||
if err != nil {
|
||||
return fmt.Errorf("stat snapshot %s: %w", source, err)
|
||||
}
|
||||
in, err := os.Open(source)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open snapshot %s: %w", source, err)
|
||||
}
|
||||
defer func() { retErr = errors.Join(retErr, in.Close()) }()
|
||||
out, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, info.Mode().Perm())
|
||||
if err != nil {
|
||||
return fmt.Errorf("create %s: %w", path, err)
|
||||
}
|
||||
defer func() { retErr = errors.Join(retErr, out.Close()) }()
|
||||
if _, err := io.Copy(out, in); err != nil {
|
||||
return fmt.Errorf("write %s: %w", path, err)
|
||||
}
|
||||
if err := out.Sync(); err != nil {
|
||||
return fmt.Errorf("sync %s: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Ownership is what makes "the State Store is stopped" a fact an offline
|
||||
// command can check, so the primitive itself is asserted here rather than only
|
||||
// through the commands that use it: exclusivity, release, and per-database
|
||||
// independence.
|
||||
func TestOwnershipIsExclusivePerDatabase(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
first := filepath.Join(dir, "archie.db-tasks.sqlite")
|
||||
second := filepath.Join(dir, "other.db-tasks.sqlite")
|
||||
|
||||
held, err := AcquireOwnership(first)
|
||||
if err != nil {
|
||||
t.Fatalf("first AcquireOwnership: %v", err)
|
||||
}
|
||||
if _, err := AcquireOwnership(first); !errors.Is(err, ErrStoreOwned) {
|
||||
t.Fatalf("second AcquireOwnership on the same database = %v, want ErrStoreOwned", err)
|
||||
}
|
||||
// A different database is a different claim: one process owning its store
|
||||
// must not stop another from owning its own.
|
||||
elsewhere, err := AcquireOwnership(second)
|
||||
if err != nil {
|
||||
t.Fatalf("AcquireOwnership on an unrelated database: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(first + ".lock"); err != nil {
|
||||
t.Fatalf("ownership left no sidecar to lock: %v", err)
|
||||
}
|
||||
|
||||
if err := held.Release(); err != nil {
|
||||
t.Fatalf("Release: %v", err)
|
||||
}
|
||||
reacquired, err := AcquireOwnership(first)
|
||||
if err != nil {
|
||||
t.Fatalf("AcquireOwnership after Release: %v", err)
|
||||
}
|
||||
if err := errors.Join(reacquired.Release(), elsewhere.Release()); err != nil {
|
||||
t.Fatalf("release on cleanup: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The snapshot an update takes is the state a rollback restores, and it is
|
||||
// restored by the release that wrote it. Verifying the copy must therefore not
|
||||
// migrate it: a snapshot silently upgraded to this binary's schema would be
|
||||
// refused by the older binary it exists to serve.
|
||||
func TestBackupSnapshotKeepsTheSchemaVersionItCopied(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
database := filepath.Join(dir, "archie.db-tasks.sqlite")
|
||||
snapshot := filepath.Join(dir, "snapshot.sqlite")
|
||||
st, err := Open(t.Context(), database)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := st.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// An older release's file: the schema version it wrote, which this binary
|
||||
// is expected to migrate when the store is next opened for serving.
|
||||
if err := setUserVersion(t.Context(), database, taskSchemaVersion-1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := Backup(t.Context(), database, snapshot); err != nil {
|
||||
t.Fatalf("Backup: %v", err)
|
||||
}
|
||||
version, err := ValidateFile(t.Context(), snapshot)
|
||||
if err != nil {
|
||||
t.Fatalf("ValidateFile on the snapshot: %v", err)
|
||||
}
|
||||
if version != taskSchemaVersion-1 {
|
||||
t.Fatalf("snapshot schema version = %d, want the %d it copied", version, taskSchemaVersion-1)
|
||||
}
|
||||
}
|
||||
|
||||
func setUserVersion(ctx context.Context, path string, version int) error {
|
||||
db, err := sql.Open("sqlite", path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = db.Close() }()
|
||||
_, err = db.ExecContext(ctx, fmt.Sprintf(`PRAGMA user_version = %d`, version))
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// ErrStoreOwned is returned when a task database is already owned by another
|
||||
// process. Ownership is what makes "the State Store is stopped" a fact an
|
||||
// offline command can check rather than a rule the operator is trusted to
|
||||
// follow: overwriting a database a running server still holds leaves it
|
||||
// writing to an unlinked inode while the store serves a file nobody owns.
|
||||
var ErrStoreOwned = errors.New("task database is owned by another process")
|
||||
|
||||
// Ownership is one process's exclusive claim over one task database, held for
|
||||
// as long as that process owns the file. The serving State Store takes it for
|
||||
// its whole life; the offline recovery commands take it for the duration of an
|
||||
// operation that rewrites the file.
|
||||
//
|
||||
// The claim lives in a sidecar file beside the database rather than on the
|
||||
// database inode itself, so it survives the rename a restore performs: after
|
||||
// the swap the sidecar is still the one path every process locks.
|
||||
type Ownership struct {
|
||||
f *os.File
|
||||
}
|
||||
|
||||
// AcquireOwnership takes the store's ownership lock without blocking. A
|
||||
// database another process owns comes back as ErrStoreOwned, never as a wait:
|
||||
// a process that must not proceed has to hear so, and a blocking lock would
|
||||
// turn a mistyped command into a hung terminal.
|
||||
func AcquireOwnership(path string) (*Ownership, error) {
|
||||
lockPath := ownershipLockPath(path)
|
||||
if err := os.MkdirAll(filepath.Dir(lockPath), 0o755); err != nil {
|
||||
return nil, fmt.Errorf("create store directory for %s: %w", lockPath, err)
|
||||
}
|
||||
f, err := os.OpenFile(lockPath, os.O_RDWR|os.O_CREATE, 0o644)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open store lock %s: %w", lockPath, err)
|
||||
}
|
||||
taken, err := lockFile(f)
|
||||
if err != nil {
|
||||
return nil, errors.Join(fmt.Errorf("lock %s: %w", lockPath, err), f.Close())
|
||||
}
|
||||
if !taken {
|
||||
return nil, errors.Join(fmt.Errorf("%s: %w", path, ErrStoreOwned), f.Close())
|
||||
}
|
||||
return &Ownership{f: f}, nil
|
||||
}
|
||||
|
||||
// Release drops the claim and closes the sidecar. Closing the descriptor alone
|
||||
// would release it too; the explicit unlock keeps the intent readable and
|
||||
// makes Release idempotent.
|
||||
func (o *Ownership) Release() error {
|
||||
if o == nil || o.f == nil {
|
||||
return nil
|
||||
}
|
||||
err := unlockFile(o.f)
|
||||
f := o.f
|
||||
o.f = nil
|
||||
return errors.Join(err, f.Close())
|
||||
}
|
||||
|
||||
// ownershipLockPath is the sidecar every owner of path locks. It is derived
|
||||
// from the database path, so the State Store process and an offline command
|
||||
// that were given the same database agree on it without configuration.
|
||||
func ownershipLockPath(path string) string { return path + ".lock" }
|
||||
@@ -0,0 +1,30 @@
|
||||
//go:build unix
|
||||
|
||||
package store
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// lockFile takes a non-blocking exclusive flock on f. The boolean reports
|
||||
// whether it was taken: a lock another process holds (EWOULDBLOCK) is a normal
|
||||
// answer to "is the State Store running?", not an error.
|
||||
func lockFile(f *os.File) (bool, error) {
|
||||
err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
|
||||
if errors.Is(err, syscall.EWOULDBLOCK) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func unlockFile(f *os.File) error {
|
||||
if err := syscall.Flock(int(f.Fd()), syscall.LOCK_UN); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
//go:build windows
|
||||
|
||||
// flock is a POSIX primitive and Windows is not a supported deployment surface
|
||||
// (CLAUDE.md's deployment model is Linux and macOS), so this file fails closed:
|
||||
// every lock attempt reports an error rather than silently degrading to "no
|
||||
// lock". An offline command that cannot prove it owns the database must refuse
|
||||
// to rewrite it, which is the only safe reading of an unavailable primitive.
|
||||
//
|
||||
// If a Windows deployment ever becomes supported, swap this for a real
|
||||
// LockFileEx/UnlockFileEx pair; nothing else in the package needs to change.
|
||||
|
||||
package store
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
)
|
||||
|
||||
func lockFile(*os.File) (bool, error) {
|
||||
return false, errors.New("store ownership locking requires a POSIX flock, which this platform does not provide")
|
||||
}
|
||||
|
||||
func unlockFile(*os.File) error { return nil }
|
||||
@@ -183,10 +183,18 @@ fi
|
||||
# that cannot take one must not start: once binaries are replaced, a failure
|
||||
# leaves the previous release unable to read the store it just migrated. This
|
||||
# runs ahead of the adapter refresh below so a refusal changes nothing at all.
|
||||
#
|
||||
# The snapshot is taken by the recovery subcommand of the release being
|
||||
# installed, from its own checkout, rather than by hand-rolled sqlite3 shell:
|
||||
# the operator running `archie-state-store backup` by hand and the installer
|
||||
# are then the same implementation, and the two cannot drift. The installed
|
||||
# binary is the previous release, which may predate the subcommand, so it
|
||||
# cannot be the one asked -- and a release whose recovery path will not build is a
|
||||
# release that cannot back up, which is exactly what this pre-flight refuses.
|
||||
if [[ ",$components," == *,daemon,* ]] && [ -n "$TASK_DB_PATH" ] && [ -f "$TASK_DB_PATH" ]; then
|
||||
echo "==> backing up task database for schema-safe rollback"
|
||||
mkdir -p "$(dirname "$TASK_DB_BACKUP")"
|
||||
if ! sqlite3 "$TASK_DB_PATH" ".backup '$TASK_DB_BACKUP'"; then
|
||||
if ! go run ./cmd/archie-state-store backup -db "$TASK_DB_PATH" -out "$TASK_DB_BACKUP"; then
|
||||
echo "refusing to install $GW: could not back up the task database at $TASK_DB_PATH, so a failed update could not be rolled back" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
Reference in new issue
Block a user