From b5b58c370ae151afad5af54264ff65ea3c4d05c3 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Mon, 21 Sep 2026 20:40:47 +1000 Subject: [PATCH] fix(statestore): read a kind the store does not hold as its seed, and stop validate touching the daemon's log Four defects from the re-run contract review, all in the recovery path. A store missing a control-plane kind was refused while a store with no resources table at all was accepted -- two stances inside validateStore for states the daemon cannot tell apart. The State Store seeds every kind from its own config before it serves (RunStateStore -> ImportConfig), so a kind the file does not hold yet is one archied reads the seed for: an upgrade that adds a kind, and a validate before the State Store has started again, exited 1 with "read provider-settings: resource not found" for a store the daemon boots on fine. StoredRuntimeConfig now reads a missing kind as the seed the State Store would write for it (Definition.seededValue, shared with ImportConfig so the two cannot drift), and the missing-table branch is the same stance taken to its extreme. defaultConfigPath() derived the daemon's config with os.UserConfigDir, which disagrees with archied's configHome on darwin and rejects a relative XDG_CONFIG_HOME outright -- leaving validate with an empty path and no verdict, on a host where archied boots from relative-dir/archie/config.toml. It is now the daemon's own rule, exported as archied.DefaultConfigPath and used by the daemon's own -config default too, with a test pinning the equality. -h exited 2, the code documented as "the command line was wrong", while the serve path in the same binary exits 0 and main.go advertises -h for these subcommands. ErrHelp now returns 0. validate resolved the boot config through loadConfig -> setupLogging, which created cfg.Log.File's directory, opened the log, and wrote a line stamped component="daemon" -- a diagnostic editing the deployment it inspects, and rotating a log the daemon is writing. boot.stderrLog keeps setupLogging on the stderr logger for the recovery path: no feed, no task-log registry, no durable copy. Tests: TestRecoveryValidateRunsTheGateBootRuns now asserts the unseeded and the missing-kind stores are accepted (both exited 1 before), while the bogus-policy case still refuses; TestRecoveryHelpExitsZero, TestRecoveryValidateLeavesTheDaemonLogAlone and TestRecoveryDefaultConfigFollowsTheDaemonsConfigHome are new. Each was run red first, and the seed fallback and stderrLog were mutation-checked. --- cmd/archie-state-store/recovery.go | 24 ++-- cmd/archie-state-store/recovery_test.go | 111 +++++++++++++++++- docs/architecture/safe-change-and-recovery.md | 2 +- internal/app/archied/bootstrap.go | 13 ++ internal/app/archied/main.go | 11 +- internal/app/archied/state_store_recovery.go | 21 ++-- internal/app/controlplane/registry.go | 39 +++++- internal/app/controlplane/runtime_config.go | 31 ++++- 8 files changed, 215 insertions(+), 37 deletions(-) diff --git a/cmd/archie-state-store/recovery.go b/cmd/archie-state-store/recovery.go index 4b537e9ff..259410236 100644 --- a/cmd/archie-state-store/recovery.go +++ b/cmd/archie-state-store/recovery.go @@ -2,25 +2,20 @@ package main import ( "context" + "errors" "flag" "fmt" "io" - "os" - "path/filepath" "github.com/samcharles93/archie-core/internal/app/archied" ) -// defaultConfigPath is the configuration the serve path reads by default; the -// offline validate asks what the daemon would boot with, so it reads the same -// one. -func defaultConfigPath() string { - base, err := os.UserConfigDir() - if err != nil { - return "" - } - return filepath.Join(base, "archie", "config.toml") -} +// defaultConfigPath is the configuration the serve path reads by default, and +// the one the daemon boots with. It is the daemon's own rule rather than a +// second derivation of it: os.UserConfigDir (what the sibling processes use) +// disagrees with archied on darwin and rejects a relative XDG_CONFIG_HOME +// outright, which would leave validate answering about a file nobody boots. +func defaultConfigPath() string { return archied.DefaultConfigPath() } // recoveryUsage is the whole flag surface of the offline recovery commands. // They are subcommands of this binary rather than a recovery binary of their @@ -77,6 +72,11 @@ func runRecovery(args []string, stdout, stderr io.Writer) int { flags.StringVar(&options.Overlay, "config-overlay", "", "configuration overlay file or directory") } if err := flags.Parse(args[1:]); err != nil { + // A requested help is not a wrong command line: the usage above is the + // answer, and the serve path exits 0 for the same request. + if errors.Is(err, flag.ErrHelp) { + return 0 + } return 2 } if flags.NArg() > 0 { diff --git a/cmd/archie-state-store/recovery_test.go b/cmd/archie-state-store/recovery_test.go index 187b89762..a775037e1 100644 --- a/cmd/archie-state-store/recovery_test.go +++ b/cmd/archie-state-store/recovery_test.go @@ -317,16 +317,94 @@ func TestRecoveryValidateRunsTheGateBootRuns(t *testing.T) { t.Errorf("refusal must name what boot rejects; stderr = %q", stderr) } - // A store the State Store never seeded: the daemon cannot layer its - // settings from it either, so the offline check must not call it valid. + // A store the State Store never seeded holds no kinds at all. Its next start + // creates the resources table and seeds every kind from this same config + // (internal/app/archied/state_store.go), so archied layers those seeds and + // starts: refusing the file here would send an operator to restore a + // snapshot for a store the daemon boots on perfectly well. unseeded := filepath.Join(dir, "unseeded.db-tasks.sqlite") openTaskStore(t, unseeded) code, stdout, stderr = runRecoveryCmd(t, "validate", "-db", unseeded, "-config", configPath) - if code == 0 { - t.Fatalf("validate accepted a store with no stored settings; stdout = %q", stdout) + if code != 0 { + t.Fatalf("validate refused a store the State Store would seed on its next start; stderr = %q", stderr) } - if !strings.Contains(stderr, "provider-settings") { - t.Errorf("refusal must name the resource kind boot cannot read; stderr = %q", stderr) + if !strings.Contains(stdout, "0 stored resources validate") { + t.Errorf("validate must report that there was nothing stored to check; stdout = %q", stdout) + } + + // The upgrade case: a release defines a kind the store does not hold yet, + // and validate runs before the State Store has started again. The State + // Store seeds the kind it is missing, so this is the same stance as the + // store with nothing stored at all -- the file is not the reason the daemon + // would refuse to start, and saying otherwise costs the operator a snapshot. + partial := filepath.Join(dir, "partial.db-tasks.sqlite") + seedStoreResources(t, openTaskStore(t, partial), configPath) + if err := execRaw(t, partial, `DELETE FROM resources WHERE kind='provider-settings'`); err != nil { + t.Fatal(err) + } + code, stdout, stderr = runRecoveryCmd(t, "validate", "-db", partial, "-config", configPath) + if code != 0 { + t.Fatalf("validate refused a store missing a kind the State Store seeds; stderr = %q", stderr) + } + if count := storedResourceCount(t, stdout); count == 0 { + t.Errorf("validate must still check the kinds the store does hold; stdout = %q", stdout) + } +} + +// The subcommands advertise -h as the way to read their flags, and the serve +// path in the same binary exits 0 for the same request, so a script probing the +// recovery surface must not read a requested help as a failure. +func TestRecoveryHelpExitsZero(t *testing.T) { + commands := []string{archied.RecoveryBackup, archied.RecoveryRestore, archied.RecoveryValidate, archied.RecoveryRollback} + for _, command := range commands { + code, _, stderr := runRecoveryCmd(t, command, "-h") + if code != 0 { + t.Errorf("%s -h exited %d, want 0; stderr = %q", command, code, stderr) + } + } +} + +// validate diagnoses a deployment; it is not the deployment. Resolving the boot +// config must not create, append to, or rotate the daemon's log file or the +// directory it lives in: a line stamped component="daemon" in archied's log is +// indistinguishable from the daemon having written it. +func TestRecoveryValidateLeavesTheDaemonLogAlone(t *testing.T) { + dir := t.TempDir() + configPath, logFile := writeConfigWithLogFile(t, dir) + db := filepath.Join(dir, "archie.db-tasks.sqlite") + seedStoreResources(t, openTaskStore(t, db), configPath) + + code, _, stderr := runRecoveryCmd(t, "validate", "-db", db, "-config", configPath) + if code != 0 { + t.Fatalf("validate exited %d: %s", code, stderr) + } + if _, err := os.Stat(logFile); !os.IsNotExist(err) { + t.Fatalf("validate touched the daemon's log file %s (stat err = %v)", logFile, err) + } + if _, err := os.Stat(filepath.Dir(logFile)); !os.IsNotExist(err) { + t.Errorf("validate created the log directory %s (stat err = %v)", filepath.Dir(logFile), err) + } +} + +// validate's verdict is the daemon's verdict only if it reads the configuration +// the daemon reads. archied resolves that default from its own configHome, which +// honours a relative XDG_CONFIG_HOME; os.UserConfigDir rejects a relative one +// outright, which left the command with no config and no verdict at all. +func TestRecoveryDefaultConfigFollowsTheDaemonsConfigHome(t *testing.T) { + relative := filepath.Join("relative", "config") + t.Setenv("XDG_CONFIG_HOME", relative) + if got, want := defaultConfigPath(), filepath.Join(relative, "archie", "config.toml"); got != want { + t.Errorf("defaultConfigPath() with a relative XDG_CONFIG_HOME = %q, want %q", got, want) + } + + absolute := t.TempDir() + t.Setenv("XDG_CONFIG_HOME", absolute) + if got, want := defaultConfigPath(), filepath.Join(absolute, "archie", "config.toml"); got != want { + t.Errorf("defaultConfigPath() = %q, want %q", got, want) + } + // The equality itself: one helper, so the two cannot drift apart again. + if got, want := defaultConfigPath(), archied.DefaultConfigPath(); got != want { + t.Errorf("defaultConfigPath() = %q, but the daemon resolves %q", got, want) } } @@ -582,6 +660,27 @@ func TestRecoveryRejectsAnUnknownSubcommand(t *testing.T) { } } +// writeConfigWithLogFile is writeMinimalConfig plus the daemon's durable log +// destination, which only the daemon may bring into existence. +func writeConfigWithLogFile(t *testing.T, dir string) (string, string) { + t.Helper() + configPath := writeMinimalConfig(t, dir) + logFile := filepath.Join(dir, "logs", "archied.log") + f, err := os.OpenFile(configPath, os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + t.Fatalf("open config to append: %v", err) + } + defer func() { + if err := f.Close(); err != nil { + t.Fatalf("close config: %v", err) + } + }() + if _, err := fmt.Fprintf(f, "\n[log]\nfile = %q\n", logFile); err != nil { + t.Fatalf("append [log]: %v", err) + } + return configPath, logFile +} + // seedStoreResources seeds the store the way the State Store seeds it at boot, // from the same config file the operator hands validate: the command's verdict // is about the pair, and a store whose settings never matched the config is a diff --git a/docs/architecture/safe-change-and-recovery.md b/docs/architecture/safe-change-and-recovery.md index d9b409f0b..a1bb55e4e 100644 --- a/docs/architecture/safe-change-and-recovery.md +++ b/docs/architecture/safe-change-and-recovery.md @@ -135,7 +135,7 @@ verdict the daemon's verdict. | --- | --- | --- | | `backup -db F -out F` | writes a transactionally consistent snapshot (`VACUUM INTO`), verified as a usable store before it is published | may be running | | `restore -db F -from F` | replaces the database with a snapshot and drops the replaced file's `-wal`/`-shm` | must be stopped | -| `validate -db F [-config C]` | runs the checks that stop archied starting: the file is readable, uncorrupted and no newer than the binary; every stored resource decodes under the definition that owns it; and `configuration.Validate` accepts the stored values layered onto `C`, the check the daemon reports as `validate database settings` | may be running | +| `validate -db F [-config C]` | runs the checks that stop archied starting: the file is readable, uncorrupted and no newer than the binary; every stored resource decodes under the definition that owns it; and `configuration.Validate` accepts the settings the daemon would read -- the stored ones, or the seed the State Store writes for a kind the file does not hold yet -- layered onto `C`, the check the daemon reports as `validate database settings` | may be running | | `rollback -db F -kind K [-revision N]` | replays an earlier revision of a stored resource through the ordinary replace, recording the rollback as a new revision | must be stopped | The two validation layers `validate` reports are not the same check, and it diff --git a/internal/app/archied/bootstrap.go b/internal/app/archied/bootstrap.go index 81d22d283..bc67a1147 100644 --- a/internal/app/archied/bootstrap.go +++ b/internal/app/archied/bootstrap.go @@ -83,6 +83,13 @@ type boot struct { cfg config.Config log *slog.Logger + // stderrLog keeps setupLogging on stderr, never touching cfg.Log.File. An + // offline command reads the bootstrap config but is not the daemon: creating + // the deployment's log, appending a line stamped component="daemon", or + // rotating a log the daemon is writing are side effects a diagnosis must not + // have, and it needs no durable copy of its own. + stderrLog bool + loader *configuration.Loader doc *configuration.Document currentProvenance atomic.Pointer[configuration.Provenance] @@ -297,6 +304,12 @@ func (b *boot) loadConfig(ctx context.Context, cfgPath, overlayPath string) erro // must not take the daemon down with it. func (b *boot) setupLogging() error { cfg := b.cfg + if b.stderrLog { + // The stderr logger newBootstrap installed is the whole destination. + // Nothing below this line is set up: the feed, the task-log registry and + // the rotating file all belong to a running daemon. + return nil + } logFeed := logging.NewFeed(1000) b.logFeed = logFeed // Task logs live alongside the store rather than under cfg.Log.File's diff --git a/internal/app/archied/main.go b/internal/app/archied/main.go index 37870ec3f..3ea12fbe2 100644 --- a/internal/app/archied/main.go +++ b/internal/app/archied/main.go @@ -315,7 +315,7 @@ type runArgs struct { } func parseArgs() (runArgs, bool) { - defaultCfg := filepath.Join(configHome(), "archie", "config.toml") + defaultCfg := DefaultConfigPath() var args runArgs flag.StringVar(&args.cfgPath, "config", defaultCfg, "path to a TOML/YAML config file or configuration directory") flag.StringVar(&args.overlayPath, "config-overlay", "", "path to a TOML/YAML overlay file or configuration directory applied on top of -config") @@ -918,6 +918,15 @@ func configHome() string { return filepath.Join(home, ".config") } +// DefaultConfigPath is the configuration the daemon reads when -config is not +// given, and the one every command that asks a question about the daemon has to +// read to be answering about the same deployment: it is the daemon's rule, not +// a second one. It is exported for exactly that reason -- an offline command +// resolving the default any other way answers about a different config file +// (os.UserConfigDir, notably, rejects a relative XDG_CONFIG_HOME outright and +// leaves the caller with no path at all). +func DefaultConfigPath() string { return filepath.Join(configHome(), "archie", "config.toml") } + // updateReportPath is where the update watchdog leaves the phase-2 outcome // of an update for this identity to relay on its next launch. The identity is // hashed so multiple identities sharing one daemon (see diff --git a/internal/app/archied/state_store_recovery.go b/internal/app/archied/state_store_recovery.go index 868755366..92b371d1f 100644 --- a/internal/app/archied/state_store_recovery.go +++ b/internal/app/archied/state_store_recovery.go @@ -132,10 +132,12 @@ func validateStore(ctx context.Context, options StateStoreRecoveryOptions) (stri defer func() { _ = st.Close() }() server := controlplane.NewServer(st) - // The resources table is only absent from a store written before the control - // plane existed. That store holds no settings to validate, and the serving - // process creates the table on its next start -- refusing it here would send - // an operator to restore a snapshot for nothing. + // Is there a resources table at all? A store written before the control plane + // existed has none, and the State Store creates it and seeds every kind from + // this same config on its next start -- so there is nothing stored to check + // with the writer's validator, and boot's gate is the config check alone. + // That is the same stance the layering takes for a kind the table is missing: + // what the daemon reads for a kind nothing was ever stored for is the seed. checked := 0 stored, err := st.StoresResources(ctx) if err != nil { @@ -153,10 +155,8 @@ func validateStore(ctx context.Context, options StateStoreRecoveryOptions) (stri } document := base if stored { - // A store the State Store never seeded has no kinds to layer, and the - // daemon cannot start against it either: the State Store seeds every kind - // from its own config at boot. - if document, _, err = server.StoredRuntimeConfig(ctx, base); err != nil { + document, _, err = server.StoredRuntimeConfig(ctx, base) + if err != nil { return "", err } } @@ -172,8 +172,13 @@ func validateStore(ctx context.Context, options StateStoreRecoveryOptions) (stri // the only recovery command that asks a question about the process rather than // the file, and that question is about the stored values layered onto this // document, so the document is part of the answer. +// +// It reads that config with the daemon's stderr logger: resolving it is a +// diagnosis, and a diagnosis that opened, appended to or rotated cfg.Log.File +// would edit the deployment it is inspecting. func bootConfig(ctx context.Context, options StateStoreRecoveryOptions) (config.Config, error) { b := newBootstrap() + b.stderrLog = true defer b.cleanup() if err := b.loadConfig(ctx, options.Config, options.Overlay); err != nil { return config.Config{}, err diff --git a/internal/app/controlplane/registry.go b/internal/app/controlplane/registry.go index bbfa206e5..629421c93 100644 --- a/internal/app/controlplane/registry.go +++ b/internal/app/controlplane/registry.go @@ -72,13 +72,9 @@ func (s *Server) ImportConfig(ctx context.Context, cfg config.Config) (map[strin if !errors.Is(err, store.ErrResourceNotFound) { return nil, err } - value, err := json.Marshal(definition.Seed(cfg)) + value, err := definition.seededValue(cfg) if err != nil { - return nil, fmt.Errorf("seed %s: %w", definition.Kind, err) - } - value, err = definition.Decode(value) - if err != nil { - return nil, fmt.Errorf("seed %s: %w", definition.Kind, err) + return nil, err } resource, err = s.store.PutResource(ctx, store.ResourceWrite{Kind: definition.Kind, Value: value, Actor: "system:migration", Source: "legacy-config", RequestID: "import:" + definition.Kind, ExpectedVersion: 0, At: time.Now().UTC()}) if err != nil { @@ -89,6 +85,37 @@ func (s *Server) ImportConfig(ctx context.Context, cfg config.Config) (map[strin return versions, nil } +// seededValue is the value the State Store writes for a kind it does not hold +// yet: the definition's seed derived from cfg, run through the same Decode a +// write applies. ImportConfig stores it; the offline config layering reads it +// for the same reason, so "what the daemon sees for a kind that was never +// stored" is one value rather than two. +func (d Definition) seededValue(cfg config.Config) ([]byte, error) { + value, err := json.Marshal(d.Seed(cfg)) + if err != nil { + return nil, fmt.Errorf("seed %s: %w", d.Kind, err) + } + value, err = d.Decode(value) + if err != nil { + return nil, fmt.Errorf("seed %s: %w", d.Kind, err) + } + return value, nil +} + +// seededValues is what the State Store would write for every kind, keyed by +// kind. +func (s *Server) seededValues(cfg config.Config) (map[string][]byte, error) { + values := make(map[string][]byte, len(s.ordered)) + for _, definition := range s.ordered { + value, err := definition.seededValue(cfg) + if err != nil { + return nil, err + } + values[definition.Kind] = value + } + return values, nil +} + // Owns reports whether kind is a resource the control plane owns, so a caller // acting on a stored value can refuse a kind nobody defines before it reaches // the store. diff --git a/internal/app/controlplane/runtime_config.go b/internal/app/controlplane/runtime_config.go index 93fb6f9d2..0bd90c6c5 100644 --- a/internal/app/controlplane/runtime_config.go +++ b/internal/app/controlplane/runtime_config.go @@ -3,11 +3,13 @@ package controlplane import ( "context" "encoding/json" + "errors" "fmt" "time" "github.com/samcharles93/archie-core/internal/config" pb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1" + "github.com/samcharles93/archie-core/internal/store" ) // resourceReader is the one read the layering performs: a resource kind's value @@ -43,19 +45,42 @@ func (c *Client) RuntimeChatConfig(ctx context.Context, base config.ChatConfig) // configuration.Validate, so an offline check of that database has to make the // same first move with the same implementation; called with the config the // daemon would load, the result is the document boot decides on. +// +// A kind the store does not hold is read as the seed the State Store would +// write for it from base (see storeReader.query), because that store seeds +// every kind before it serves. func (s *Server) StoredRuntimeConfig(ctx context.Context, base config.Config) (config.Config, map[string]int64, error) { - return runtimeConfigFrom(ctx, storeReader{resources: s.store}, base) + seeds, err := s.seededValues(base) + if err != nil { + return config.Config{}, nil, err + } + return runtimeConfigFrom(ctx, storeReader{resources: s.store, seeds: seeds}, base) } // storeReader reads the server's own store. It mirrors what the gRPC read -// answers -- a missing kind is an error, not an empty value -- so a store the -// daemon cannot layer settings from is one this reports rather than skips. +// answers -- a kind that is stored is the value and version it holds -- with +// one difference: a kind the store does not hold yet is the value the State +// Store seeds for it, since the daemon dials a store that has already run +// ImportConfig over the same config. type storeReader struct { resources ResourceStore + seeds map[string][]byte } func (r storeReader) query(ctx context.Context, kind string, decode func([]byte) error) (int64, error) { resource, err := r.resources.Resource(ctx, kind) + if errors.Is(err, store.ErrResourceNotFound) { + seed, ok := r.seeds[kind] + if !ok { + return 0, fmt.Errorf("read %s: %w", kind, store.ErrResourceNotFound) + } + if err := decode(seed); err != nil { + return 0, fmt.Errorf("decode %s seed: %w", kind, err) + } + // No version: nothing has been stored for this kind, so nothing has + // been applied either. + return 0, nil + } if err != nil { return 0, fmt.Errorf("read %s: %w", kind, err) }