diff --git a/cmd/archie-state-store/recovery.go b/cmd/archie-state-store/recovery.go index 00ee75750..4b537e9ff 100644 --- a/cmd/archie-state-store/recovery.go +++ b/cmd/archie-state-store/recovery.go @@ -5,10 +5,23 @@ import ( "flag" "fmt" "io" + "os" + "path/filepath" "github.com/samcharles93/archie-core/internal/app/archied" ) +// defaultConfigPath is the configuration the serve path reads by default; the +// offline validate asks what the daemon would boot with, so it reads the same +// one. +func defaultConfigPath() string { + base, err := os.UserConfigDir() + if err != nil { + return "" + } + return filepath.Join(base, "archie", "config.toml") +} + // recoveryUsage is the whole flag surface of the offline recovery commands. // They are subcommands of this binary rather than a recovery binary of their // own: offline database maintenance belongs to the process that owns the file @@ -25,7 +38,8 @@ owns it, while backup takes a consistent snapshot of a serving store. backup -db FILE -out FILE write a snapshot of the database restore -db FILE -from FILE replace the database with a snapshot - validate -db FILE check the database the way boot does + validate -db FILE [-config FILE] check the file, its stored settings and the + configuration the daemon would boot with rollback -db FILE -kind KIND replay an earlier revision of a stored [-revision N] resource through the ordinary replace ` @@ -56,6 +70,12 @@ func runRecovery(args []string, stdout, stderr io.Writer) int { flags.StringVar(&options.From, "from", "", "snapshot file restore reads") flags.StringVar(&options.Kind, "kind", "", "control-plane resource kind rollback replays") flags.Int64Var(&options.Revision, "revision", 0, "revision rollback replays (default: the newest one older than the current value)") + // Only validate asks a question about the process rather than the file, so + // only validate takes the configuration the process would boot with. + if command == archied.RecoveryValidate { + flags.StringVar(&options.Config, "config", defaultConfigPath(), "configuration file or directory the daemon boots with") + flags.StringVar(&options.Overlay, "config-overlay", "", "configuration overlay file or directory") + } if err := flags.Parse(args[1:]); err != nil { return 2 } diff --git a/cmd/archie-state-store/recovery_test.go b/cmd/archie-state-store/recovery_test.go index efe96c4f9..187b89762 100644 --- a/cmd/archie-state-store/recovery_test.go +++ b/cmd/archie-state-store/recovery_test.go @@ -18,6 +18,7 @@ import ( "fmt" "os" "path/filepath" + "strconv" "strings" "testing" @@ -25,6 +26,7 @@ import ( "github.com/samcharles93/archie-core/internal/app/archied" "github.com/samcharles93/archie-core/internal/app/controlplane" + "github.com/samcharles93/archie-core/internal/infrastructure/configuration" "github.com/samcharles93/archie-core/internal/store" ) @@ -142,6 +144,28 @@ func TestRecoveryBackupRefusesAMissingDatabase(t *testing.T) { } } +// backup is the one recovery command allowed to run against a serving store, +// and that exemption is only sound while it never replaces the database. With +// -out naming the database it renames the snapshot over the live file and +// leaves the WAL of the database it just unlinked -- the state the restore path +// refuses up front and the code elsewhere calls unrecoverable. +func TestRecoveryBackupRefusesTheDatabaseAsItsOwnSnapshot(t *testing.T) { + db := filepath.Join(t.TempDir(), "archie.db-tasks.sqlite") + st := openTaskStore(t, db) + seedTask(t, st, "live") + + code, _, stderr := runRecoveryCmd(t, "backup", "-db", db, "-out", db) + if code == 0 { + t.Fatal("backup over the database itself succeeded") + } + if !strings.Contains(stderr, "is the database itself") { + t.Errorf("refusal must say the snapshot is the database; stderr = %q", stderr) + } + if got := taskTitles(t, db); len(got) != 1 || got[0] != "live" { + t.Fatalf("refused backup changed the database: tasks = %v", got) + } +} + // Restoring is the forward-migration escape hatch: the previous release // reads this file, so it must be exactly the snapshot, with the WAL that // belongs to the replaced database gone rather than replayed over it. @@ -242,18 +266,68 @@ func TestRecoveryRestoreRefusesTheDatabaseAsItsOwnSnapshot(t *testing.T) { func TestRecoveryValidateAcceptsAHealthyStore(t *testing.T) { dir := t.TempDir() + configPath := writeMinimalConfig(t, dir) db := filepath.Join(dir, "archie.db-tasks.sqlite") st := openTaskStore(t, db) seedTask(t, st, "healthy") - seedResource(t, st, controlplane.ModelRoleAssignmentsKind, `{"implement":"anthropic/claude"}`, "seed") + seedStoreResources(t, st, configPath) - code, stdout, stderr := runRecoveryCmd(t, "validate", "-db", db) + code, stdout, stderr := runRecoveryCmd(t, "validate", "-db", db, "-config", configPath) if code != 0 { t.Fatalf("validate exited %d: %s", code, stderr) } if !strings.Contains(stdout, "schema version") { t.Errorf("validate must report the schema version it verified; stdout = %q", stdout) } + // The count is part of what the operator reads, and a check that reports + // "0 stored resources" for a store full of them is worse than no check. + if count := storedResourceCount(t, stdout); count == 0 { + t.Errorf("validate reported no stored resources for a seeded store; stdout = %q", stdout) + } +} + +// validate's verdict is only useful if it is the daemon's verdict. Boot refuses +// on configuration.Validate over the file config with every stored resource +// layered onto it (internal/app/archied/control_plane.go), not on the write +// path's own per-resource decode, and the two disagree: the write path accepts +// a scheduling policy boot rejects. +func TestRecoveryValidateRunsTheGateBootRuns(t *testing.T) { + dir := t.TempDir() + configPath := writeMinimalConfig(t, dir) + db := filepath.Join(dir, "archie.db-tasks.sqlite") + seedStoreResources(t, openTaskStore(t, db), configPath) + + // The control: a store seeded from this config is one the daemon boots on. + code, _, stderr := runRecoveryCmd(t, "validate", "-db", db, "-config", configPath) + if code != 0 { + t.Fatalf("validate exited %d on a store the daemon starts on: %s", code, stderr) + } + + // A hand-edited value, or an older revision of one: the write path's own + // validator never inspects dispatch.trigger, so only the config gate + // catches it, and the daemon exits 1 on it (validateDispatch). + if err := execRaw(t, db, `UPDATE resources SET value='{"poll_interval":"1m","max_retries":3,"dispatch":{"trigger":"bogus"}}' WHERE kind='scheduling-policy'`); err != nil { + t.Fatal(err) + } + code, stdout, stderr := runRecoveryCmd(t, "validate", "-db", db, "-config", configPath) + if code == 0 { + t.Fatalf("validate accepted a stored policy the daemon refuses to boot with; stdout = %q", stdout) + } + if !strings.Contains(stderr, "dispatch.trigger") { + t.Errorf("refusal must name what boot rejects; stderr = %q", stderr) + } + + // A store the State Store never seeded: the daemon cannot layer its + // settings from it either, so the offline check must not call it valid. + unseeded := filepath.Join(dir, "unseeded.db-tasks.sqlite") + openTaskStore(t, unseeded) + code, stdout, stderr = runRecoveryCmd(t, "validate", "-db", unseeded, "-config", configPath) + if code == 0 { + t.Fatalf("validate accepted a store with no stored settings; stdout = %q", stdout) + } + if !strings.Contains(stderr, "provider-settings") { + t.Errorf("refusal must name the resource kind boot cannot read; stderr = %q", stderr) + } } // validate exists to answer "would archied start against this file". Each case @@ -318,16 +392,19 @@ func TestRecoveryValidateRefusesWhatTheDaemonRefuses(t *testing.T) { }) t.Run("stored_resource_that_will_not_validate", func(t *testing.T) { - // The stored settings value that stops archied starting: it can only - // get here by bypassing the write path, which is exactly the state the - // control plane fails closed on and the operator has no path back from. + // A stored value that both gates refuse: the write path stops it at + // Decode (workflow.ExecutionSettings.Validate rejects a negative limit), + // and boot stops on it too -- startWorkflowExecutionSettings decodes the + // same document and exits 1. It can only get here by bypassing the write + // path, which is exactly the state the operator has no path back from. db := newStore(t) - seedResource(t, openTaskStore(t, db), controlplane.ModelRoleAssignmentsKind, `{"implement":"claude"}`, "hand-edited") + seedResource(t, openTaskStore(t, db), controlplane.WorkflowExecutionSettingsKind, + `{"max_model_tool_steps":-1,"max_runtime_seconds":60,"max_consecutive_gate_failures":3}`, "hand-edited") code, _, stderr := runRecoveryCmd(t, "validate", "-db", db) if code == 0 { t.Fatal("validate accepted a stored resource the write path would refuse") } - if !strings.Contains(stderr, controlplane.ModelRoleAssignmentsKind) { + if !strings.Contains(stderr, controlplane.WorkflowExecutionSettingsKind) { t.Errorf("refusal must name the resource kind; stderr = %q", stderr) } }) @@ -351,11 +428,14 @@ func TestRecoveryValidateRefusesWhatTheDaemonRefuses(t *testing.T) { // settings. The serving process creates the table on its next start, so // refusing this file would send an operator to restore a snapshot for a // database that would have started perfectly well. - db := newStore(t) + dir := t.TempDir() + configPath := writeMinimalConfig(t, dir) + db := filepath.Join(dir, "archie.db-tasks.sqlite") + openTaskStore(t, db) if err := execRaw(t, db, `DROP TABLE resources`); err != nil { t.Fatal(err) } - code, stdout, stderr := runRecoveryCmd(t, "validate", "-db", db) + code, stdout, stderr := runRecoveryCmd(t, "validate", "-db", db, "-config", configPath) if code != 0 { t.Fatalf("validate exited %d: %s", code, stderr) } @@ -502,6 +582,36 @@ func TestRecoveryRejectsAnUnknownSubcommand(t *testing.T) { } } +// seedStoreResources seeds the store the way the State Store seeds it at boot, +// from the same config file the operator hands validate: the command's verdict +// is about the pair, and a store whose settings never matched the config is a +// state boot does not recognise either. +func seedStoreResources(t *testing.T, st *store.Store, configPath string) { + t.Helper() + doc, err := configuration.New(nil).Resolve(configPath, "") + if err != nil { + t.Fatalf("resolve %s: %v", configPath, err) + } + if _, err := controlplane.NewServer(st).ImportConfig(t.Context(), doc.Config); err != nil { + t.Fatalf("seed store resources: %v", err) + } +} + +// storedResourceCount reads the count validate reports out of its summary line. +func storedResourceCount(t *testing.T, stdout string) int { + t.Helper() + before, found := strings.CutSuffix(stdout, " stored resources validate\n") + if !found { + t.Fatalf("no resource count in %q", stdout) + } + fields := strings.Fields(before) + count, err := strconv.Atoi(fields[len(fields)-1]) + if err != nil { + t.Fatalf("resource count in %q: %v", stdout, err) + } + return count +} + // seedResource writes one revision of a control-plane resource through the // store, the way the control plane's replace does. The request ID is derived // from the label and the revision it produces: the store deduplicates a diff --git a/docs/architecture/safe-change-and-recovery.md b/docs/architecture/safe-change-and-recovery.md index 6f10f70ff..d9b409f0b 100644 --- a/docs/architecture/safe-change-and-recovery.md +++ b/docs/architecture/safe-change-and-recovery.md @@ -127,15 +127,23 @@ Nothing skips the control plane at boot, and the only writer for a resource is the daemon fails closed without. `archie-state-store` therefore carries the offline recovery commands (`archie-core-6zw0`). They work on the task database file directly -- the configured `db_path` with `-tasks.sqlite` appended, not -`db_path` itself -- and need neither `archied` nor the Web UI. +`db_path` itself -- and need neither `archied` nor the Web UI running: `validate` +reads the configuration file the daemon boots with, which is what makes its +verdict the daemon's verdict. | Command | What it does | State Store | | --- | --- | --- | | `backup -db F -out F` | writes a transactionally consistent snapshot (`VACUUM INTO`), verified as a usable store before it is published | may be running | | `restore -db F -from F` | replaces the database with a snapshot and drops the replaced file's `-wal`/`-shm` | must be stopped | -| `validate -db F` | checks the file the way boot meets it: readable, not corrupt, schema no newer than the binary, and every stored resource decoding under the definition that owns it | must be stopped | +| `validate -db F [-config C]` | runs the checks that stop archied starting: the file is readable, uncorrupted and no newer than the binary; every stored resource decodes under the definition that owns it; and `configuration.Validate` accepts the stored values layered onto `C`, the check the daemon reports as `validate database settings` | may be running | | `rollback -db F -kind K [-revision N]` | replays an earlier revision of a stored resource through the ordinary replace, recording the rollback as a new revision | must be stopped | +The two validation layers `validate` reports are not the same check, and it +reports both rather than merging them: the write path's own decode is stricter +about shape (it rejects unknown fields boot's decode ignores), while boot is +stricter about meaning (only boot checks `dispatch.trigger`, and a positive +`poll_interval`). Either one refusing is a store somebody has to look at. + No rollback RPC exists and none is needed: a revision carries its own value, so restoring one is the same replace the dashboard and the channels already use. The offline command is that operation with the store stopped -- the one state in @@ -144,12 +152,14 @@ which the daemon's own API cannot serve it. "Must be stopped" is checked rather than assumed. The serving process takes an exclusive flock on `.lock` for its whole life, and `restore` and `rollback` refuse with `task database is owned by another process` while another process -holds it. `backup` is the deliberate exception: the update installer runs inside -the process it is replacing and cannot stop the State Store, so the snapshot -that makes a failed update reversible is taken against a serving store -- and -`scripts/archie-update-install` takes it through this subcommand rather than -through its own `sqlite3` shell, so the installer and an operator run one -implementation. +holds it. The read-only commands -- `backup` and `validate` -- take no lock and +are meant to run against a serving store: `backup` because the update installer +runs inside the process it is replacing and cannot stop the State Store, and +`validate` because an operator checks a running deployment with it. That +`backup` exemption is only sound while backup never replaces the database, so +naming the database as its own `-out` is refused. `scripts/archie-update-install` +takes its pre-flight snapshot through this subcommand rather than through its +own `sqlite3` shell, so the installer and an operator run one implementation. `rollback` is the surgical route back from a bad stored value; `restore` is the blunt one, and is also the way back from a schema migration a release cannot diff --git a/internal/app/archied/state_store_recovery.go b/internal/app/archied/state_store_recovery.go index bf0acc93d..868755366 100644 --- a/internal/app/archied/state_store_recovery.go +++ b/internal/app/archied/state_store_recovery.go @@ -20,7 +20,9 @@ import ( "time" "github.com/samcharles93/archie-core/internal/app/controlplane" + "github.com/samcharles93/archie-core/internal/config" pb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1" + "github.com/samcharles93/archie-core/internal/infrastructure/configuration" "github.com/samcharles93/archie-core/internal/store" ) @@ -53,6 +55,12 @@ type StateStoreRecoveryOptions struct { Out string // From is the snapshot restore reads. From string + // Config and Overlay are the configuration the daemon would boot with. + // validate asks whether the stored settings are a state archied starts on, + // and that question is about the stored values *layered onto this + // configuration*, so the file config is part of the answer. + Config string + Overlay string // Kind is the control-plane resource kind rollback replays. Kind string // Revision is the revision rollback replays. Zero means the newest @@ -84,7 +92,7 @@ func RunStateStoreRecovery(ctx context.Context, options StateStoreRecoveryOption if options.DB == "" { return "", errors.New("validate requires -db") } - return validateStore(ctx, options.DB) + return validateStore(ctx, options) case RecoveryRollback: if options.DB == "" || options.Kind == "" { return "", errors.New("rollback requires -db and -kind") @@ -95,20 +103,35 @@ func RunStateStoreRecovery(ctx context.Context, options StateStoreRecoveryOption } } -// validateStore answers "would archied start against this file". The file-level -// checks come first, so the store is only read once its schema is one this -// binary understands; the resource check then decodes every stored value with -// the definition that owns it, which is the same validation a write applies. -func validateStore(ctx context.Context, path string) (string, error) { - version, err := store.ValidateFile(ctx, path) +// validateStore answers the question an operator actually has: would archied +// start against this file. It runs the checks that stop it, in the order boot +// meets them. +// +// First the file itself, since a database that is corrupt, newer than this +// binary, or not a database is one the serving process refuses to open. Then +// the writer's own per-resource validation, which is what a hand-edited value +// violated. Then boot's gate: the file config with every stored resource +// layered onto it, and configuration.Validate -- the check whose failure the +// daemon reports as "validate database settings". +// +// The two validation layers disagree in both directions, and both are reported +// rather than merged. The write path is stricter about shape (it rejects +// unknown fields boot's decode ignores), while boot is stricter about meaning +// (only boot checks dispatch.trigger and a positive poll interval, both of +// which the write path's own validators let through). A store either check +// refuses is a store somebody has to look at. +func validateStore(ctx context.Context, options StateStoreRecoveryOptions) (string, error) { + version, err := store.ValidateFile(ctx, options.DB) if err != nil { return "", err } - st, err := store.OpenReadOnly(ctx, path) + st, err := store.OpenReadOnly(ctx, options.DB) if err != nil { return "", err } defer func() { _ = st.Close() }() + + server := controlplane.NewServer(st) // The resources table is only absent from a store written before the control // plane existed. That store holds no settings to validate, and the serving // process creates the table on its next start -- refusing it here would send @@ -119,12 +142,43 @@ func validateStore(ctx context.Context, path string) (string, error) { return "", err } if stored { - checked, err = controlplane.NewServer(st).ValidateStored(ctx) - if err != nil { + if checked, err = server.ValidateStored(ctx); err != nil { return "", err } } - return fmt.Sprintf("%s is a valid store at schema version %d; %d stored resources validate", path, version, checked), nil + + base, err := bootConfig(ctx, options) + if err != nil { + return "", err + } + document := base + if stored { + // A store the State Store never seeded has no kinds to layer, and the + // daemon cannot start against it either: the State Store seeds every kind + // from its own config at boot. + if document, _, err = server.StoredRuntimeConfig(ctx, base); err != nil { + return "", err + } + } + if err := configuration.Validate(&document); err != nil { + // The daemon's own wording, so an operator can match this failure to the + // one that stopped archied starting. + return "", fmt.Errorf("validate database settings: %w", err) + } + return fmt.Sprintf("%s is a valid store at schema version %d; %d stored resources validate", options.DB, version, checked), nil +} + +// bootConfig resolves the configuration the daemon would boot with. validate is +// the only recovery command that asks a question about the process rather than +// the file, and that question is about the stored values layered onto this +// document, so the document is part of the answer. +func bootConfig(ctx context.Context, options StateStoreRecoveryOptions) (config.Config, error) { + b := newBootstrap() + defer b.cleanup() + if err := b.loadConfig(ctx, options.Config, options.Overlay); err != nil { + return config.Config{}, err + } + return b.cfg, nil } // rollbackResource is the one operation the Web UI cannot be asked to perform diff --git a/internal/app/controlplane/registry.go b/internal/app/controlplane/registry.go index ddf66142a..bbfa206e5 100644 --- a/internal/app/controlplane/registry.go +++ b/internal/app/controlplane/registry.go @@ -102,10 +102,13 @@ func (s *Server) Owns(kind string) bool { // resources it checked. Errors name the kind and the revision it was stored at, // which is what an operator needs to roll the value back. // -// This is how an offline validate reproduces the refusal that stops archied -// starting: the check is the writer's own, so it cannot drift from what a -// replace would accept, and a value can only be stored unvalidated by bypassing -// the write path entirely. +// It is the write path's own check, not boot's: boot refuses on +// configuration.Validate over the stored values layered onto the file config, +// and the two disagree in both directions (this one rejects unknown fields +// boot's decode ignores, while only boot checks dispatch.trigger and a positive +// poll interval). A caller answering "would the daemon start" therefore needs +// StoredRuntimeConfig and configuration.Validate as well -- see +// validateStore in internal/app/archied/state_store_recovery.go. func (s *Server) ValidateStored(ctx context.Context) (int, error) { checked := 0 var failures []error diff --git a/internal/app/controlplane/runtime_config.go b/internal/app/controlplane/runtime_config.go index c98f0d86e..93fb6f9d2 100644 --- a/internal/app/controlplane/runtime_config.go +++ b/internal/app/controlplane/runtime_config.go @@ -10,6 +10,15 @@ import ( pb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1" ) +// resourceReader is the one read the layering performs: a resource kind's value +// and the version it came from. The gRPC client and the in-process server both +// provide it, so "what the database covers in the running config" has exactly +// one implementation. The offline validate runs the same code as boot, which is +// the only way its verdict can be boot's verdict. +type resourceReader interface { + query(ctx context.Context, kind string, decode func([]byte) error) (int64, error) +} + // RuntimeConfig applies restart-scoped database resources over bootstrap // configuration. Values intentionally absent from a control-plane projection, // such as secret-bearing MCP headers and channel update commands, remain @@ -19,9 +28,47 @@ import ( // layers them in can report which version it is running // (docs/prds/control-plane-apply-status.md). func (c *Client) RuntimeConfig(ctx context.Context, base config.Config) (config.Config, map[string]int64, error) { + return runtimeConfigFrom(ctx, c, base) +} + +// RuntimeChatConfig layers the stored channel settings over the file document's +// chat section. +func (c *Client) RuntimeChatConfig(ctx context.Context, base config.ChatConfig) (config.ChatConfig, int64, error) { + return runtimeChatConfigFrom(ctx, c, base) +} + +// StoredRuntimeConfig is RuntimeConfig's layering over the store this server +// owns, for a caller holding the database file rather than a connection to it. +// Boot layers the stored settings onto the file config and then runs +// configuration.Validate, so an offline check of that database has to make the +// same first move with the same implementation; called with the config the +// daemon would load, the result is the document boot decides on. +func (s *Server) StoredRuntimeConfig(ctx context.Context, base config.Config) (config.Config, map[string]int64, error) { + return runtimeConfigFrom(ctx, storeReader{resources: s.store}, base) +} + +// storeReader reads the server's own store. It mirrors what the gRPC read +// answers -- a missing kind is an error, not an empty value -- so a store the +// daemon cannot layer settings from is one this reports rather than skips. +type storeReader struct { + resources ResourceStore +} + +func (r storeReader) query(ctx context.Context, kind string, decode func([]byte) error) (int64, error) { + resource, err := r.resources.Resource(ctx, kind) + if err != nil { + return 0, fmt.Errorf("read %s: %w", kind, err) + } + if err := decode(resource.Value); err != nil { + return 0, fmt.Errorf("decode %s: %w", kind, err) + } + return resource.Version, nil +} + +func runtimeConfigFrom(ctx context.Context, reader resourceReader, base config.Config) (config.Config, map[string]int64, error) { out := base.Clone() versions := map[string]int64{} - if err := c.queryInto(ctx, versions, ProviderSettingsKind, func(value []byte) error { + if err := layerResource(ctx, reader, versions, ProviderSettingsKind, func(value []byte) error { var providers map[string]providerDocument if err := json.Unmarshal(value, &providers); err != nil { return err @@ -34,18 +81,18 @@ func (c *Client) RuntimeConfig(ctx context.Context, base config.Config) (config. }); err != nil { return config.Config{}, nil, err } - if err := c.queryJSONInto(ctx, versions, ModelRoleAssignmentsKind, &out.Models); err != nil { + if err := layerResourceJSON(ctx, reader, versions, ModelRoleAssignmentsKind, &out.Models); err != nil { return config.Config{}, nil, err } - if err := c.queryJSONInto(ctx, versions, RepositoryPoliciesKind, &out.Repos); err != nil { + if err := layerResourceJSON(ctx, reader, versions, RepositoryPoliciesKind, &out.Repos); err != nil { return config.Config{}, nil, err } - chat, chatVersion, err := c.RuntimeChatConfig(ctx, out.Chat) + chat, chatVersion, err := runtimeChatConfigFrom(ctx, reader, out.Chat) if err != nil { return config.Config{}, nil, err } out.Chat, versions[ChannelSettingsKind] = chat, chatVersion - if err := c.queryInto(ctx, versions, SchedulingPolicyKind, func(value []byte) error { + if err := layerResource(ctx, reader, versions, SchedulingPolicyKind, func(value []byte) error { var policy schedulingPolicy if err := json.Unmarshal(value, &policy); err != nil { return err @@ -59,11 +106,11 @@ func (c *Client) RuntimeConfig(ctx context.Context, base config.Config) (config. }); err != nil { return config.Config{}, nil, err } - return c.runtimeToolConfig(ctx, versions, out) + return runtimeToolConfigFrom(ctx, reader, versions, out) } -func (c *Client) runtimeToolConfig(ctx context.Context, versions map[string]int64, out config.Config) (config.Config, map[string]int64, error) { - if err := c.queryInto(ctx, versions, ToolSettingsKind, func(value []byte) error { +func runtimeToolConfigFrom(ctx context.Context, reader resourceReader, versions map[string]int64, out config.Config) (config.Config, map[string]int64, error) { + if err := layerResource(ctx, reader, versions, ToolSettingsKind, func(value []byte) error { var settings toolSettings if err := json.Unmarshal(value, &settings); err != nil { return err @@ -81,7 +128,7 @@ func (c *Client) runtimeToolConfig(ctx context.Context, versions map[string]int6 }); err != nil { return config.Config{}, nil, err } - if err := c.queryInto(ctx, versions, PluginSettingsKind, func(value []byte) error { + if err := layerResource(ctx, reader, versions, PluginSettingsKind, func(value []byte) error { var settings pluginSettings if err := json.Unmarshal(value, &settings); err != nil { return err @@ -91,15 +138,15 @@ func (c *Client) runtimeToolConfig(ctx context.Context, versions map[string]int6 }); err != nil { return config.Config{}, nil, err } - if err := c.queryJSONInto(ctx, versions, ContainerRuntimePoliciesKind, &out.Containers); err != nil { + if err := layerResourceJSON(ctx, reader, versions, ContainerRuntimePoliciesKind, &out.Containers); err != nil { return config.Config{}, nil, err } return out, versions, nil } -func (c *Client) RuntimeChatConfig(ctx context.Context, base config.ChatConfig) (config.ChatConfig, int64, error) { +func runtimeChatConfigFrom(ctx context.Context, reader resourceReader, base config.ChatConfig) (config.ChatConfig, int64, error) { out := base - version, err := c.query(ctx, ChannelSettingsKind, func(value []byte) error { + version, err := reader.query(ctx, ChannelSettingsKind, func(value []byte) error { var settings channelSettings if err := json.Unmarshal(value, &settings); err != nil { return err @@ -117,10 +164,10 @@ func (c *Client) RuntimeChatConfig(ctx context.Context, base config.ChatConfig) return out, version, err } -// queryInto decodes a resource and records the version it came from, so -// RuntimeConfig ends up holding the version of every kind it layered in. -func (c *Client) queryInto(ctx context.Context, versions map[string]int64, kind string, decode func([]byte) error) error { - version, err := c.query(ctx, kind, decode) +// layerResource decodes a resource and records the version it came from, so the +// layering ends up holding the version of every kind it applied. +func layerResource(ctx context.Context, reader resourceReader, versions map[string]int64, kind string, decode func([]byte) error) error { + version, err := reader.query(ctx, kind, decode) if err != nil { return err } @@ -128,8 +175,8 @@ func (c *Client) queryInto(ctx context.Context, versions map[string]int64, kind return nil } -func (c *Client) queryJSONInto(ctx context.Context, versions map[string]int64, kind string, target any) error { - return c.queryInto(ctx, versions, kind, func(value []byte) error { return json.Unmarshal(value, target) }) +func layerResourceJSON(ctx context.Context, reader resourceReader, versions map[string]int64, kind string, target any) error { + return layerResource(ctx, reader, versions, kind, func(value []byte) error { return json.Unmarshal(value, target) }) } // query returns the version of the resource it decoded, so callers that layer diff --git a/internal/store/maintenance.go b/internal/store/maintenance.go index 12c3af029..d1165a600 100644 --- a/internal/store/maintenance.go +++ b/internal/store/maintenance.go @@ -133,6 +133,16 @@ func Backup(ctx context.Context, database, snapshot string) error { if err := RequireDatabase(database); err != nil { return err } + // The exemption from the ownership lock below is only sound while backup + // never replaces the database. With -out naming the database it would + // rename the snapshot over a file a serving State Store may still have + // open, orphaning the WAL of the file it just unlinked -- so it is refused + // exactly as Restore refuses it. + if same, err := sameFile(database, snapshot); err != nil { + return err + } else if same { + return fmt.Errorf("snapshot %s is the database itself", snapshot) + } if err := os.MkdirAll(filepath.Dir(snapshot), 0o755); err != nil { return fmt.Errorf("create snapshot directory: %w", err) } @@ -163,9 +173,11 @@ func Backup(ctx context.Context, database, snapshot string) error { // Restore replaces database with the snapshot at snapshot. The snapshot is // verified before anything is destroyed, so a file that is not a usable store -// cannot take the database down with it, and the database is replaced by a -// rename, so a failure leaves either the old file or the new one rather than a -// mixture. +// cannot take the database down with it, and the database file itself is +// replaced by a rename, so a failure leaves either the old file or the new one +// rather than a mixture. The replaced file's -wal/-shm are removed first, which +// the comment at that step explains: between the two calls the database is +// merely incomplete, and re-running the command completes it. // // It holds the ownership lock for its duration: rewriting a database a running // State Store still has open would leave that process writing to an unlinked