From fad7cdc92e44982855ee81ec034543decb4183c9 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Mon, 21 Sep 2026 20:11:31 +1000 Subject: [PATCH 1/4] fix(config): keep the fields a file overlay omits inside a map entry Loader.overlayFile decoded the overlay file straight into the config the base file produced, and a decode replaces a map-valued entry wholesale: [services.state] target = ... in an overlay cleared the target_token the base file set. That is archie-core-e2e2 on the user-visible -config-overlay path -- deployments/dev.toml is the documented argument and sets [services.state] target/listen, while target_token lives in the installed config.toml. An enabled image.hosted entry lost its class and api_key_env the same way, silently turning the provider off. apply.go's foldOverrides is the fix that path was missing, so the file overlay now goes through the same fold as the map overlay: the overlay file parses to its raw mapping (decodeFileMapping), the fields of each partially addressed map entry are folded forward out of the base config, and the completed mapping is decoded over it. Loader.ApplyOverlay, ApplyOverlayValues and apply.go stay: the hazard they were written for is this one. An overlay file still reports its own unknown keys, from a separate decode into a scratch value. --- .../infrastructure/configuration/apply.go | 40 ++++-- .../infrastructure/configuration/decode.go | 26 ++++ .../infrastructure/configuration/loader.go | 17 ++- .../configuration/loader_test.go | 124 ++++++++++++++++++ .../configuration/unknown_keys_test.go | 27 ++++ 5 files changed, 223 insertions(+), 11 deletions(-) diff --git a/internal/infrastructure/configuration/apply.go b/internal/infrastructure/configuration/apply.go index f73402989..fa39ffe49 100644 --- a/internal/infrastructure/configuration/apply.go +++ b/internal/infrastructure/configuration/apply.go @@ -27,16 +27,36 @@ func ApplyOverlayValues(cfg *config.Config, overrides map[string]any) error { if len(overrides) == 0 { return nil } - // yaml gives a struct field the field-level treatment promised above, but a - // map VALUE is decoded into a fresh zero value: an entry the overlay only - // partially addresses would lose every field it does not name, so an - // operator changing services.state.target alone would silently drop - // target_token. Fold the fields the overlay omits forward out of the entry - // cfg already holds, leaving the decode a complete entry to read. Doing it - // here, once, is what makes the promise hold for every map of structs - // (services, image.hosted, image.local, providers) instead of at each - // field's reader. - folded, err := foldOverrides(reflect.ValueOf(cfg), overrides) + return applyOverlayMapping(cfg, overrides) +} + +// applyOverlayFile layers the overlay file at path over cfg, with the same +// field-level precedence ApplyOverlayValues gives a map overlay. The file +// overlay path needs it for the reason recorded on foldOverrides: decoding an +// overlay into cfg replaces a map-valued entry wholesale, so a file naming +// only services.state.target cleared the target_token the base file set. Both +// overlay paths go through the fold, so neither can drift from the other. +func applyOverlayFile(path string, cfg *config.Config) error { + doc, err := decodeFileMapping(path) + if err != nil { + return err + } + return applyOverlayMapping(cfg, doc) +} + +// applyOverlayMapping is the decode both overlay paths share: the fields each +// map-valued entry the overlay does not name are folded forward out of the +// entry cfg already holds, and the completed mapping is then decoded over cfg. +// +// yaml gives a struct field the field-level treatment ApplyOverlayValues +// promises, but a map VALUE is decoded into a fresh zero value: an entry the +// overlay only partially addresses would lose every field it does not name, so +// an operator changing services.state.target alone would silently drop +// target_token. Folding here, once, is what makes the promise hold for every +// map of structs (services, image.hosted, image.local, providers) instead of +// at each field's reader. +func applyOverlayMapping(cfg *config.Config, doc map[string]any) error { + folded, err := foldOverrides(reflect.ValueOf(cfg), doc) if err != nil { return err } diff --git a/internal/infrastructure/configuration/decode.go b/internal/infrastructure/configuration/decode.go index 79fd0b387..08975ef44 100644 --- a/internal/infrastructure/configuration/decode.go +++ b/internal/infrastructure/configuration/decode.go @@ -27,6 +27,32 @@ func decodeConfigFileKeys(path string, target any) ([]string, error) { } } +// decodeFileMapping parses a configuration file into the nested mapping it +// carries, with no typed target. An overlay needs this shape: a typed decode +// cannot distinguish a key the file omits from a key it sets to the zero +// value, and that distinction is what carrying the fields of a partially +// addressed map entry forward is built on (see applyFileOverlay). +func decodeFileMapping(path string) (map[string]any, error) { + var doc map[string]any + switch filepath.Ext(path) { + case ".yaml", ".yml": + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("%w: reading %s: %w", ErrUnreadable, path, err) + } + if err := yaml.Unmarshal(data, &doc); err != nil { + return nil, fmt.Errorf("%w: parsing %s: %w", ErrUnreadable, path, err) + } + case ".toml": + if _, err := toml.DecodeFile(path, &doc); err != nil { + return nil, fmt.Errorf("%w: parsing %s: %w", ErrUnreadable, path, err) + } + default: + return nil, fmt.Errorf("%w: config file %s must end in .toml, .yaml, or .yml", ErrUnreadable, path) + } + return doc, nil +} + // decodeTOMLKeys decodes a TOML file into target, additionally // returning the top-level dotted key paths present in the file that // target did not consume (toml.MetaData.Undecoded()). A single decode diff --git a/internal/infrastructure/configuration/loader.go b/internal/infrastructure/configuration/loader.go index d64e52fb5..9fbb3d461 100644 --- a/internal/infrastructure/configuration/loader.go +++ b/internal/infrastructure/configuration/loader.go @@ -173,7 +173,13 @@ func (l *Loader) overlayFile(basePath, overlayPath string) (*Document, error) { doc.Provenance.record(Origin{Path: basePath, Role: RoleMain, Layer: LayerBase}) if overlayPath != "" { - cfgKeys, err = decodeConfigFileKeys(overlayPath, &doc.Config) + // The overlay is folded over the base config rather than decoded into + // it: a decode replaces a map-valued entry wholesale, clearing the + // fields of that entry the overlay does not name (applyFileOverlay). + if err := applyOverlayFile(overlayPath, &doc.Config); err != nil { + return nil, err + } + cfgKeys, err = overlayFileKeys(overlayPath) if err != nil { return nil, err } @@ -188,6 +194,15 @@ func (l *Loader) overlayFile(basePath, overlayPath string) (*Document, error) { return l.finalize(doc) } +// overlayFileKeys reports the top-level keys of an overlay file that the +// config decode does not consume. The file itself reaches the config through +// applyOverlayFile, which needs its raw mapping; this typed decode exists only +// for the key report, which only TOML's decoder can produce. +func overlayFileKeys(path string) ([]string, error) { + var scratch config.Config + return decodeConfigFileKeys(path, &scratch) +} + // unknownKeys returns the keys present in both a and b: one config file // feeds more than one decode target (&doc.Config and &doc.Scheduling), so // a key legitimately owned by one target decodes cleanly there and only diff --git a/internal/infrastructure/configuration/loader_test.go b/internal/infrastructure/configuration/loader_test.go index caba94334..f37b003fe 100644 --- a/internal/infrastructure/configuration/loader_test.go +++ b/internal/infrastructure/configuration/loader_test.go @@ -571,6 +571,130 @@ func TestOverlay(t *testing.T) { } } +// TestResolveFileOverlayPreservesOmittedMapEntryFields characterizes the +// -config-overlay path at the depth the runtime overlay is already guarded at +// (TestApplyOverlayValuesPreservesOmittedNestedFields). +// +// It goes through Loader.Resolve, the production entry point, because the +// hazard is a property of how the overlay file is applied, not of any one +// decode helper: Loader.overlayFile decodes the overlay into the very config +// the base file produced, and a map-valued entry is replaced wholesale, so +// every field of that entry the overlay does not name is cleared. +// +// That is archie-core-e2e2 on a user-visible path. deployments/dev.toml is the +// documented -config-overlay argument and sets [services.state] target/listen, +// while the installed config.toml is where [services.state].target_token lives: +// the overlay silently drops the token. +func TestResolveFileOverlayPreservesOmittedMapEntryFields(t *testing.T) { + tests := []struct { + name string + base string + overlayName string + overlay string + check func(t *testing.T, cfg config.Config) + }{ + { + // The deployment shape quoted above, with the value that is read + // only when a client dials a non-loopback State Store. + name: "services.state entry keeps the token it does not name", + base: "[services.state]\ntarget = \"127.0.0.1:9090\"\ntarget_token = \"secret\"\n", + overlay: "[services.state]\ntarget = \"10.0.0.5:9090\"\n", + check: func(t *testing.T, cfg config.Config) { + t.Helper() + got := cfg.Services[config.ServiceNameState] + if got.Target != "10.0.0.5:9090" { + t.Errorf("services.state.target = %q, want the overlay's 10.0.0.5:9090", got.Target) + } + if got.TargetToken != "secret" { + t.Errorf("services.state.target_token = %q, want the base's secret: a file overlay must not clear a field it does not name", got.TargetToken) + } + }, + }, + { + // An enabled hosted provider must keep its class and key env: the + // overlay naming only base_url would otherwise clear them and fail + // validation after the load. + name: "image.hosted entry keeps the fields it does not name", + base: "[image.hosted.minimax]\nenabled = true\nclass = \"minimax\"\n" + + "api_key_env = \"MINIMAX_API_KEY\"\n", + overlay: "[image.hosted.minimax]\nbase_url = \"https://api.example\"\n", + check: func(t *testing.T, cfg config.Config) { + t.Helper() + got := cfg.Image.Hosted["minimax"] + want := config.ImageHostedProvider{ + Enabled: true, + Class: "minimax", + APIKeyEnv: "MINIMAX_API_KEY", + BaseURL: "https://api.example", + } + if got != want { + t.Errorf("image.hosted.minimax = %+v, want %+v", got, want) + } + }, + }, + { + name: "providers entry keeps the fields it does not name", + base: "[providers.anthropic]\nclass = \"anthropic\"\n" + + "api_key_env = \"ANTHROPIC_API_KEY\"\n", + overlay: "[providers.anthropic]\nbase_url = \"https://proxy.example\"\n", + check: func(t *testing.T, cfg config.Config) { + t.Helper() + got := cfg.Providers["anthropic"] + want := config.Provider{ + Class: "anthropic", + APIKeyEnv: "ANTHROPIC_API_KEY", + BaseURL: "https://proxy.example", + } + if got != want { + t.Errorf("providers.anthropic = %+v, want %+v", got, want) + } + }, + }, + { + // The overlay file may be YAML (-config-overlay accepts either + // format), and it is folded by the same code. + name: "yaml overlay keeps the token it does not name", + base: "[services.state]\ntarget = \"127.0.0.1:9090\"\ntarget_token = \"secret\"\n", + overlayName: "dev.yaml", + overlay: "services:\n state:\n target: 10.0.0.5:9090\n", + check: func(t *testing.T, cfg config.Config) { + t.Helper() + got := cfg.Services[config.ServiceNameState] + if got.Target != "10.0.0.5:9090" { + t.Errorf("services.state.target = %q, want the overlay's 10.0.0.5:9090", got.Target) + } + if got.TargetToken != "secret" { + t.Errorf("services.state.target_token = %q, want the base's secret", got.TargetToken) + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + basePath := filepath.Join(dir, "config.toml") + overlayName := tt.overlayName + if overlayName == "" { + overlayName = "dev.toml" + } + overlayPath := filepath.Join(dir, overlayName) + if err := os.WriteFile(basePath, []byte(minimalValidConfigTOML+tt.base), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(overlayPath, []byte(tt.overlay), 0o600); err != nil { + t.Fatal(err) + } + + doc, err := New(nil).Resolve(basePath, overlayPath) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + tt.check(t, doc.Config) + }) + } +} + func TestLoadBytesDoesNotExist(t *testing.T) { _, err := loadBytes(nil) if err == nil { diff --git a/internal/infrastructure/configuration/unknown_keys_test.go b/internal/infrastructure/configuration/unknown_keys_test.go index f3ad8dc97..7896e664c 100644 --- a/internal/infrastructure/configuration/unknown_keys_test.go +++ b/internal/infrastructure/configuration/unknown_keys_test.go @@ -89,6 +89,33 @@ func TestUnknownSchedulingKeyIsStillReported(t *testing.T) { } } +// TestOverlayUnknownKeysAreStillReported pins that the overlay path kept its +// drift detection when the overlay file started being applied through the +// config fold (applyOverlayFile) instead of being decoded into the document's +// config: the key report comes from a separate decode, so losing it would be +// silent. +func TestOverlayUnknownKeysAreStillReported(t *testing.T) { + dir := t.TempDir() + basePath := filepath.Join(dir, "config.toml") + overlayPath := filepath.Join(dir, "dev.toml") + if err := os.WriteFile(basePath, []byte(minimalValidConfigTOML), 0o600); err != nil { + t.Fatal(err) + } + // typo: concurrancy. Unknown to both decode targets, so it is a real + // unknown key rather than Hazard 1's cross-target case. + if err := os.WriteFile(overlayPath, []byte("[containers]\nmax_concurrancy = 4\n"), 0o600); err != nil { + t.Fatal(err) + } + + doc, err := New(nil).Resolve(basePath, overlayPath) + if err != nil { + t.Fatalf("Resolve: %v (an unknown key must not fail the load)", err) + } + if !slices.Contains(doc.UnknownKeys, "containers.max_concurrancy") { + t.Errorf("UnknownKeys = %v, want it to contain %q", doc.UnknownKeys, "containers.max_concurrancy") + } +} + // TestExampleConfigHasNoUnknownKeys is the contract test the design // promised: the checked-in template must never itself trip the detector. func TestExampleConfigHasNoUnknownKeys(t *testing.T) { From 13efe4fd1a78163512640c2ea9c52b022030c09f Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Mon, 21 Sep 2026 20:18:01 +1000 Subject: [PATCH 2/4] fix(config): stop judging database-owned settings on the file load path Loader.finalize ran the whole validation over the file document, so a stale TOML value in a setting the control plane owns failed startup -- including archie-state-store, which resolves its file config and opens the store without ever layering a database resource over it. docs/prds/runtime-control-plane.md, "Bootstrap, migration, and recovery": after migration, settings in TOML are ignored and cannot block State Store startup. The checks over the settings a control plane resource is seeded from and replaces wholesale now run only on the effective document: providers (provider-settings), poll_interval and dispatch (scheduling-policy), containers (container-runtime-policies) and the repository lists (repository-policies). They moved from validate to validateDatabaseOwned; validateBootstrap keeps the checks over settings no resource carries (forge intake, identities, nats, memory, image, capture). File sources pass validateBootstrap and Loader.ApplyOverlay, which layers overrides onto an effective document, passes Validate. Nothing stops being judged: the daemon and the Gateway both run the full Validate in boot.runtimeConfig, the one place database resources are layered in, before any capability starts -- so a stored value that will not validate still stops the process rather than being published. Cases that asserted the load path rejects a database-owned value now assert both layers, and TestRuntimeConfigFailsClosedOnAnInvalidDatabaseValue pins the second. --- .../archied/control_plane_validation_test.go | 82 +++++++++ internal/app/archied/reload_test.go | 6 +- .../infrastructure/configuration/loader.go | 21 ++- .../configuration/loader_test.go | 155 ++++++++++++++++-- .../infrastructure/configuration/validate.go | 110 ++++++++++--- .../configuration/validate_test.go | 6 + 6 files changed, 334 insertions(+), 46 deletions(-) create mode 100644 internal/app/archied/control_plane_validation_test.go diff --git a/internal/app/archied/control_plane_validation_test.go b/internal/app/archied/control_plane_validation_test.go new file mode 100644 index 000000000..1a28fbf4a --- /dev/null +++ b/internal/app/archied/control_plane_validation_test.go @@ -0,0 +1,82 @@ +package archied + +import ( + "errors" + "log/slog" + "testing" + + "github.com/samcharles93/archie-core/internal/app/controlplane" + "github.com/samcharles93/archie-core/internal/infrastructure/configuration" +) + +// TestLoadConfigAcceptsStaleDatabaseOwnedValues is archie-core-i3qm at the boot +// entry point. docs/prds/runtime-control-plane.md, "Bootstrap, migration, and +// recovery": after migration, settings in TOML are ignored and cannot block +// State Store startup. archie-state-store boots through loadConfig and then +// opens the store; it never layers a control-plane resource over the file +// document (state_store.go), so a stale value in a setting the control plane +// owns must not fail boot. Every value below is one a control-plane resource +// replaces wholesale. +func TestLoadConfigAcceptsStaleDatabaseOwnedValues(t *testing.T) { + tests := []struct { + name string + body string + }{ + { + name: "provider base_url carrying userinfo", + body: "[providers.openai]\nclass = \"openai\"\nbase_url = \"https://token@example.com/v1\"\n", + }, + { + name: "negative poll_interval", + body: "poll_interval = \"-5s\"\n", + }, + { + name: "unrecognised dispatch.trigger", + body: "[dispatch]\ntrigger = \"labels\"\n", + }, + { + name: "negative containers.volume_ttl", + body: "[containers]\nvolume_ttl = \"-1m\"\n", + }, + { + name: "malformed repos test_glob", + body: "[[repos]]\nowner = \"acme\"\nname = \"app\"\ntest_glob = \"[\"\n", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + path := t.TempDir() + "/config.toml" + writeConfig(t, path, "bot_user = \"widget\"\n"+tt.body) + + b := &boot{log: slog.New(slog.DiscardHandler)} + if err := b.loadConfig(t.Context(), path, ""); err != nil { + t.Fatalf("loadConfig: %v (a stale database-owned value must not fail boot)", err) + } + }) + } +} + +// TestRuntimeConfigFailsClosedOnAnInvalidDatabaseValue is the other half of the +// same change: the checks that no longer run at load run on the effective +// document, at the one place database resources are layered in. A stored value +// that will not validate must stop archied starting rather than be published as +// the running configuration -- a state nothing can recover from in band. +func TestRuntimeConfigFailsClosedOnAnInvalidDatabaseValue(t *testing.T) { + resources := databaseOwnedResources() + resources[controlplane.ProviderSettingsKind] = map[string]any{ + "main": map[string]any{"class": "openai", "base_url": "https://token@example.com/v1"}, + } + b := newReloadBoot(t, &controlPlaneStub{values: resources}) + + err := b.loadRuntimeConfig(t.Context()) + if err == nil { + t.Fatal("loadRuntimeConfig accepted a database value that does not validate") + } + if !errors.Is(err, configuration.ErrInvalidInput) { + t.Errorf("loadRuntimeConfig = %v, want it to wrap configuration.ErrInvalidInput", err) + } + if got := b.cfgHolder.Get().Providers["main"].BaseURL; got == "https://token@example.com/v1" { + t.Error("the invalid database value was published as the running configuration") + } +} diff --git a/internal/app/archied/reload_test.go b/internal/app/archied/reload_test.go index cd4072483..0490bf3a8 100644 --- a/internal/app/archied/reload_test.go +++ b/internal/app/archied/reload_test.go @@ -30,8 +30,12 @@ func minimalConfigTOML(botUser string) string { return "bot_user = \"" + botUser + "\"\n[[repos]]\nowner = \"acme\"\nname = \"app\"\n" } +// invalidConfigTOML carries a value the bootstrap document still owns, so +// resolving it fails. The settings the control plane owns are no longer judged +// at load, so using one of those here would silently turn these tests into a +// false pass. func invalidConfigTOML() string { - return "bot_user = \"widget\"\ndispatch = { trigger = \"bogus-trigger\" }\n[[repos]]\nowner = \"acme\"\nname = \"app\"\n" + return "bot_user = \"widget\"\n[memory]\nengine = \"not-a-real-engine\"\n[[repos]]\nowner = \"acme\"\nname = \"app\"\n" } func TestReloadAppliesAndPublishes(t *testing.T) { diff --git a/internal/infrastructure/configuration/loader.go b/internal/infrastructure/configuration/loader.go index 9fbb3d461..beb64085a 100644 --- a/internal/infrastructure/configuration/loader.go +++ b/internal/infrastructure/configuration/loader.go @@ -146,7 +146,7 @@ func (l *Loader) ApplyOverlay(doc *Document, overrides map[string]any) (*Documen next.Config = doc.Config.Clone() next.Provenance.Origins = append([]Origin(nil), doc.Provenance.Origins...) if len(overrides) == 0 { - return l.finalize(&next) + return l.finalize(&next, Validate) } if err := ApplyOverlayValues(&next.Config, overrides); err != nil { return nil, err @@ -155,7 +155,7 @@ func (l *Loader) ApplyOverlay(doc *Document, overrides map[string]any) (*Documen return nil, err } next.Provenance.record(Origin{Path: "config_overlay (runtime)", Role: RoleMain, Layer: LayerOverlay}) - return l.finalize(&next) + return l.finalize(&next, Validate) } func (l *Loader) overlayFile(basePath, overlayPath string) (*Document, error) { @@ -191,7 +191,7 @@ func (l *Loader) overlayFile(basePath, overlayPath string) (*Document, error) { doc.Provenance.record(Origin{Path: overlayPath, Role: RoleMain, Layer: LayerOverlay}) } - return l.finalize(doc) + return l.finalize(doc, validateBootstrap) } // overlayFileKeys reports the top-level keys of an overlay file that the @@ -290,7 +290,7 @@ func (l *Loader) Dir(baseDir, overlayDir string) (*Document, error) { } } - return l.finalize(doc) + return l.finalize(doc, validateBootstrap) } // loadDir discovers and decodes one directory into doc. The base layer must @@ -351,12 +351,19 @@ func (l *Loader) decodeMain(doc *Document, path string, isYAMLFile bool) error { } // finalize applies defaults, then validates. The order matters: validation -// judges the effective configuration, including values the operator never +// judges the defaulted configuration, including values the operator never // wrote. -func (l *Loader) finalize(doc *Document) (*Document, error) { +// +// validate is a parameter because the two kinds of caller produce different +// documents. Every file source ([Loader.File], [Loader.Overlay], [Loader.Dir]) +// produces a BOOTSTRAP document and passes validateBootstrap: the settings the +// control plane owns are layered over it later, so a stale value in one of them +// must not fail the load. [Loader.ApplyOverlay] layers overrides onto a +// document that is already effective, so it passes [Validate]. +func (l *Loader) finalize(doc *Document, validate func(*config.Config) error) (*Document, error) { doc.UnknownKeys = sortedUnique(append(doc.UnknownKeys, unregisteredServiceKeys(doc.Config.Services)...)) l.applyDefaults(&doc.Config) - if err := Validate(&doc.Config); err != nil { + if err := validate(&doc.Config); err != nil { return nil, err } l.log.Debug("configuration loaded", "sources", doc.Provenance.String()) diff --git a/internal/infrastructure/configuration/loader_test.go b/internal/infrastructure/configuration/loader_test.go index f37b003fe..b8c1a7bdc 100644 --- a/internal/infrastructure/configuration/loader_test.go +++ b/internal/infrastructure/configuration/loader_test.go @@ -236,31 +236,80 @@ func TestLoadTelegramTokenSecretRefAndLegacyFallback(t *testing.T) { } } +// TestLoadRejectsInvalidConfigEnumsAndGlobs pins which layer rejects each kind +// of invalid value. Every file source produces a BOOTSTRAP document, and the +// settings the control plane owns are layered over it later; a stale TOML value +// in one of those must not fail the load (docs/prds/runtime-control-plane.md, +// "Bootstrap, migration, and recovery"). So a check over a database-owned +// setting is asserted on the effective document instead -- the same document +// boot.runtimeConfig validates after the merge. Each case asserts both halves +// it applies to, so a check that quietly disappeared fails the case rather than +// passing it. func TestLoadRejectsInvalidConfigEnumsAndGlobs(t *testing.T) { tests := []struct { - name string - extra string + name string + body string + wantLoadErr bool }{ - {name: "dispatch trigger", extra: "\n[dispatch]\ntrigger = \"labels\"\n"}, - {name: "provider userinfo", extra: "\n[providers.openai]\nclass = \"openai\"\nbase_url = \"https://token@example.com/v1\"\n"}, - {name: "provider query secret", extra: "\n[providers.openai]\nclass = \"openai\"\nbase_url = \"https://example.com/v1?api_key=secret\"\n"}, - {name: "test glob", extra: "\n[[repos]]\nowner = \"acme\"\nname = \"app\"\ntest_glob = \"[\"\n"}, + { + name: "dispatch trigger", + body: fileConfigPrefix + "[dispatch]\ntrigger = \"labels\"\n", + }, + { + name: "provider userinfo", + body: fileConfigPrefix + "[providers.openai]\nclass = \"openai\"\nbase_url = \"https://token@example.com/v1\"\n", + }, + { + name: "provider query secret", + body: fileConfigPrefix + "[providers.openai]\nclass = \"openai\"\nbase_url = \"https://example.com/v1?api_key=secret\"\n", + }, + { + name: "test glob", + body: fileConfigPrefix + "[[repos]]\nowner = \"acme\"\nname = \"app\"\ntest_glob = \"[\"\n", + }, + { + // File-owned settings: no control-plane resource carries them, so + // the load path is the layer that has to reject them. + name: "forge intake", + body: fileConfigPrefix + "[forge]\nintake = \"not-a-real-intake\"\n", + wantLoadErr: true, + }, + { + name: "memory engine", + body: fileConfigPrefix + "[memory]\nengine = \"not-a-real-engine\"\n", + wantLoadErr: true, + }, + { + name: "negative capture retention", + body: fileConfigPrefix + "[capture]\nretention = \"-1h\"\n", + wantLoadErr: true, + }, + { + name: "image default naming no provider", + body: fileConfigPrefix + "[image]\ndefault = \"not-a-real-provider\"\n", + wantLoadErr: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { path := filepath.Join(t.TempDir(), "config.toml") - contents := "bot_user = \"widget\"\n" - if tt.name != "test glob" { - contents += "\n[[repos]]\nowner = \"acme\"\nname = \"app\"\n" - } - contents += tt.extra - if err := os.WriteFile(path, []byte(contents), 0o600); err != nil { + if err := os.WriteFile(path, []byte(tt.body), 0o600); err != nil { t.Fatal(err) } - if _, err := loadFile(path); err == nil { - t.Fatal("loadFile() succeeded, want validation error") + doc, err := New(nil).File(path) + if tt.wantLoadErr { + if err == nil { + t.Fatal("File() = nil error, want the bootstrap document rejected") + } + return + } + if err != nil { + t.Fatalf("File: %v (a database-owned value must not fail the load)", err) + } + if err := Validate(&doc.Config); err == nil { + t.Fatal("Validate(effective document) = nil, want the value rejected there") } }) } @@ -487,6 +536,12 @@ func TestLoadContainerVolumeTTL(t *testing.T) { } } +// TestLoadRejectsNegativeContainerVolumeTTL: containers are a control-plane +// resource (container-runtime-policies), so a negative volume_ttl is judged on +// the effective document rather than on the file the load path reads -- a stale +// TOML value in a database-owned setting must not fail a process's startup. The +// second half of the assertion is what keeps dropping the check from passing +// this test. func TestLoadRejectsNegativeContainerVolumeTTL(t *testing.T) { path := filepath.Join(t.TempDir(), "config.toml") contents := "bot_user = \"widget\"\n" + @@ -497,8 +552,12 @@ func TestLoadRejectsNegativeContainerVolumeTTL(t *testing.T) { t.Fatal(err) } - if _, err := loadFile(path); err == nil { - t.Fatal("loadFile() accepted a negative containers.volume_ttl") + doc, err := New(nil).File(path) + if err != nil { + t.Fatalf("File: %v (a database-owned value must not fail the load)", err) + } + if err := Validate(&doc.Config); err == nil { + t.Fatal("Validate(effective document) accepted a negative containers.volume_ttl") } } @@ -695,6 +754,70 @@ func TestResolveFileOverlayPreservesOmittedMapEntryFields(t *testing.T) { } } +// TestLoadAcceptsStaleDatabaseOwnedValues is the bootstrap half of the layer +// split (archie-core-i3qm). docs/prds/runtime-control-plane.md, +// "Bootstrap, migration, and recovery": after migration, settings in TOML are +// ignored and cannot block State Store startup. archie-state-store never layers +// a database resource over its file document -- state_store.go:78 resolves the +// file and goes straight to opening the store -- so a stale value in a setting +// the control plane owns must not fail the load. +// +// Each case is a setting with a control-plane resource behind it, seeded from +// that field by controlplane.Server.ImportConfig: providers +// (provider-settings), poll_interval and dispatch (scheduling-policy), +// containers (container-runtime-policies), repositories +// (repository-policies). The second half of the assertion is what keeps this +// from being a weaker gate: the effective document, judged after the control +// plane has layered its own value in, must still reject the value. +func TestLoadAcceptsStaleDatabaseOwnedValues(t *testing.T) { + tests := []struct { + name string + body string + }{ + { + name: "providers base_url carries userinfo", + body: fileConfigPrefix + "[providers.openai]\nclass = \"openai\"\nbase_url = \"https://token@example.com/v1\"\n", + }, + { + name: "negative poll_interval", + body: fileConfigPrefix + "poll_interval = \"-5s\"\n", + }, + { + name: "unrecognised dispatch.trigger", + body: fileConfigPrefix + "[dispatch]\ntrigger = \"labels\"\n", + }, + { + name: "negative containers.volume_ttl", + body: fileConfigPrefix + "[containers]\nvolume_ttl = \"-1m\"\n", + }, + { + name: "malformed repos test_glob", + body: fileConfigPrefix + "[[repos]]\nowner = \"acme\"\nname = \"app\"\ntest_glob = \"[\"\n", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.toml") + if err := os.WriteFile(path, []byte(tt.body), 0o600); err != nil { + t.Fatal(err) + } + + doc, err := New(nil).File(path) + if err != nil { + t.Fatalf("File: %v (a stale database-owned value must not fail the load)", err) + } + if err := Validate(&doc.Config); err == nil { + t.Fatal("Validate(effective document) = nil, want the control plane's own layer to reject it") + } + }) + } +} + +// fileConfigPrefix is the smallest document the load path accepts on its own, +// so each case above adds one stale setting and nothing else. +const fileConfigPrefix = "bot_user = \"widget\"\nwork_dir = \"/base/work\"\n" + func TestLoadBytesDoesNotExist(t *testing.T) { _, err := loadBytes(nil) if err == nil { diff --git a/internal/infrastructure/configuration/validate.go b/internal/infrastructure/configuration/validate.go index 3a1a82a95..61d6eef61 100644 --- a/internal/infrastructure/configuration/validate.go +++ b/internal/infrastructure/configuration/validate.go @@ -37,9 +37,23 @@ var ( memoryEngines = []string{memoryEngineBuiltin} ) -// Validate runs the same checks Loader applies before accepting a config, -// against a config.Config value already built in memory -- e.g. by archied -// setup, before it has written anything to disk. +// Validate judges a configuration a process will run with -- every check that +// can reject one -- against a config.Config value already built in memory, e.g. +// by archied setup before it has written anything to disk. It is a superset of +// what [Loader] applies to a file (validateBootstrap): a check over a setting +// the control plane owns belongs here and not there, so that a stale TOML value +// cannot fail a process's startup. +// +// It judges an EFFECTIVE document: the bootstrap file with every setting the +// control plane owns layered over it. That layering happens at exactly one +// place, boot.runtimeConfig (internal/app/archied/control_plane.go), which runs +// this afterwards; the daemon and the Gateway both go through it, and the +// readiness config probe re-runs it against the running value. A database value +// that will not validate therefore stops the process rather than starting it +// degraded. +// +// A plain file document is judged by validateBootstrap instead: see the split +// there for why the loader must not apply this to it. // // Like validate, it does not apply defaults. Several checks (dispatch.trigger, // forge.type) only pass once a default has been filled @@ -58,31 +72,34 @@ func Validate(cfg *config.Config) error { // validate reports the first problem that would stop the daemon running. // It does not modify cfg -- run applyDefaults first. func validate(cfg *config.Config) error { - if err := validateDispatch(cfg); err != nil { + if err := validateBootstrap(cfg); err != nil { return err } + return validateDatabaseOwned(cfg) +} + +// validateBootstrap reports the first problem in the settings a bootstrap +// document still owns: the file config a process reads before it can reach the +// State Store. [Loader] applies this to every file source. +// +// It deliberately omits the settings the control plane stores, because a stale +// TOML value in one of them must not be able to fail a process's startup +// (docs/prds/runtime-control-plane.md, "Bootstrap, migration, and recovery": +// after migration, settings in TOML are ignored and cannot block State Store +// startup). archie-state-store resolves its file config and opens the store +// without ever layering a database resource over it (state_store.go), so a +// check left here is a check that can still block it. Those checks live in +// validateDatabaseOwned and run on the effective document instead. +func validateBootstrap(cfg *config.Config) error { if err := validateForgeIntake(cfg); err != nil { return err } - if err := validateProviders(cfg.Providers); err != nil { - return err - } - if len(cfg.Identities) > 0 { - if err := validateIdentities(cfg.Identities); err != nil { - return err - } - } else if err := validateSingleIdentity(cfg); err != nil { - return err - } - if err := validatePollInterval(cfg); err != nil { + if err := validateIdentityStructure(cfg); err != nil { return err } if err := validateNATS(cfg); err != nil { return err } - if err := validateContainers(cfg); err != nil { - return err - } if err := validateMemory(cfg); err != nil { return err } @@ -92,6 +109,30 @@ func validate(cfg *config.Config) error { return validateCapture(cfg) } +// validateDatabaseOwned reports the first problem in a setting the control +// plane owns. Each check below judges a field that a control-plane resource is +// seeded from and replaces wholesale: providers (provider-settings), +// poll_interval and dispatch (scheduling-policy), containers +// (container-runtime-policies), and the repository lists +// (repository-policies). It runs on the effective document, after that +// replacement, which is the only point at which the value being judged is the +// value a process will use. +func validateDatabaseOwned(cfg *config.Config) error { + if err := validateDispatch(cfg); err != nil { + return err + } + if err := validateProviders(cfg.Providers); err != nil { + return err + } + if err := validatePollInterval(cfg); err != nil { + return err + } + if err := validateContainers(cfg); err != nil { + return err + } + return validateRepositoryContents(cfg) +} + // validateImage rejects an enabled hosted provider with no class or no // resolvable credential, and a Default naming a provider that is not both // present and enabled -- either would let /image silently fall back to @@ -237,6 +278,18 @@ func validateProviders(providers map[string]config.Provider) error { return nil } +// validateIdentityStructure judges the shape of the identity definitions: the +// fields a process needs to know which identities exist and which credentials +// each one commits and calls with. The repository lists inside them are judged +// by validateRepositoryContents, because repositories are a control-plane +// resource that replaces the file's list after the merge. +func validateIdentityStructure(cfg *config.Config) error { + if len(cfg.Identities) == 0 { + return validateSingleIdentity(cfg) + } + return validateIdentities(cfg.Identities) +} + func validateIdentities(identities []config.IdentityConfig) error { for i, id := range identities { if id.Name == "" { @@ -254,9 +307,6 @@ func validateIdentities(identities []config.IdentityConfig) error { if !ForgeDisabled(id.Forge.Type) && id.Forge.Token == (config.SecretRef{}) { return fmt.Errorf("%w: identities[%d].forge.token is required (each identity needs its own secret reference; unlike the top-level [forge], there is no default)", ErrInvalidInput, i) } - if err := validateRepos(id.Repos); err != nil { - return fmt.Errorf("identities[%d]: %w", i, err) - } } return nil } @@ -278,7 +328,23 @@ func validateSingleIdentity(cfg *config.Config) error { if !oneOf(cfg.Forge.Type, forgeTypes) { return fmt.Errorf("%w: forge.type %q (want %s)", ErrInvalidInput, cfg.Forge.Type, list(forgeTypes)) } - return validateRepos(cfg.Repos) + return nil +} + +// validateRepositoryContents judges the repositories themselves, whichever list +// they appear in. Repository policies are a control-plane resource seeded from +// these, so the contents are verified on the effective document -- the one the +// database's list has already replaced the file's copy in. +func validateRepositoryContents(cfg *config.Config) error { + if len(cfg.Identities) == 0 { + return validateRepos(cfg.Repos) + } + for i, id := range cfg.Identities { + if err := validateRepos(id.Repos); err != nil { + return fmt.Errorf("identities[%d]: %w", i, err) + } + } + return nil } func validateRepos(repos []config.Repo) error { diff --git a/internal/infrastructure/configuration/validate_test.go b/internal/infrastructure/configuration/validate_test.go index 412ab1957..b7176a713 100644 --- a/internal/infrastructure/configuration/validate_test.go +++ b/internal/infrastructure/configuration/validate_test.go @@ -69,6 +69,12 @@ func TestValidate_IsCallableOutsideThePackage(t *testing.T) { } } +// TestValidate_RejectsTheSameProblemsAsLoaderLoad pins the direction that still +// holds after archie-core-i3qm split the bootstrap checks from the effective +// ones: every problem the load path rejects, Validate rejects too. Validate +// additionally rejects the settings the control plane owns, which the load path +// must let through (validateBootstrap) so a stale TOML value cannot fail a +// process's startup. The name is kept from when the two check sets were equal. func TestValidate_RejectsTheSameProblemsAsLoaderLoad(t *testing.T) { tests := []struct { name string From 042a86ce4af5f7cc86176e4902036e0a4f0dd5c7 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Mon, 21 Sep 2026 20:39:45 +1000 Subject: [PATCH 3/4] fix(config): judge a database-owned setting at the layer that owns it Review of the two previous commits found the split was incomplete in both directions: the checks moved off the file load path were still enforced anywhere the value could enter the system, and the fold reached only one of the three overlay paths. A stale database-owned TOML value could still stop archie-state-store. The process seeds every control-plane resource from the same file config on a fresh database, and the seed runs the resource's own validator; three of the five values the loader now accepts aborted the boot there (verified by booting RunStateStore before the fix). A seed that does not validate is now SKIPPED and reported instead of fatal: the resource stays absent, so the file's value is the one in effect and editing config.toml is still the fix -- the daemon and the Gateway are where it fails closed, at boot.runtimeConfig. The resource validators were weaker than configuration.Validate for exactly the fields the split reclassified, so a value the file path used to refuse at load became storable in the database, and then un-fixable by editing config.toml. The trigger enum and the test-glob rule now come from the configuration package rather than being restated, the container image is required there as it is in the file, and a table test ties the two layers together value by value. The directory form of -config-overlay kept the archie-core-e2e2 failure: it decoded each overlay file straight into the populated config, so a map-valued entry it only partly addressed still cleared the fields it did not name. Both the overlay layer's main file and its feature files now fold, which is what Loader.Dir's documented parity with Loader.Overlay has always claimed. The overlay file's key report is derived from the apply's own view of the file as well as the file decode's, because TOML matches keys case-insensitively and the fold's yaml decode does not: a key only the file decode matched was consumed by the report and dropped by the apply, a silent no-op that worked in the base file. Tests: the state-store boot is asserted against the real process, the directory overlay and the case-mismatched key each have one, and ImportConfig pins that a refused seed is not stored. --- cmd/archie-state-store/recovery_test.go | 10 +- .../archied/control_plane_validation_test.go | 136 ++++++++++++---- internal/app/archied/state_store.go | 13 +- .../app/controlplane/controlplane_test.go | 61 ++++++- .../app/controlplane/operational_settings.go | 20 +++ internal/app/controlplane/registry.go | 38 ++++- .../app/controlplane/validators_agree_test.go | 150 ++++++++++++++++++ .../infrastructure/configuration/apply.go | 60 +++++-- .../infrastructure/configuration/decode.go | 22 ++- .../infrastructure/configuration/loader.go | 98 ++++++++++-- .../configuration/loader_test.go | 127 ++++++++++++--- .../configuration/unknown_keys_test.go | 65 +++++--- .../infrastructure/configuration/validate.go | 53 +++++-- 13 files changed, 710 insertions(+), 143 deletions(-) create mode 100644 internal/app/controlplane/validators_agree_test.go diff --git a/cmd/archie-state-store/recovery_test.go b/cmd/archie-state-store/recovery_test.go index b58629f29..0f0359b31 100644 --- a/cmd/archie-state-store/recovery_test.go +++ b/cmd/archie-state-store/recovery_test.go @@ -708,9 +708,17 @@ func seedStoreResources(t *testing.T, st *store.Store, configPath string) { if err != nil { t.Fatalf("build control plane server: %v", err) } - if _, err := server.ImportConfig(t.Context(), doc.Config); err != nil { + _, skipped, err := server.ImportConfig(t.Context(), doc.Config) + if err != nil { t.Fatalf("seed store resources: %v", err) } + if len(skipped) > 0 { + // This helper stands in for the State Store's own seeding, which seeds + // every kind it can and leaves the rest absent. A skipped kind here + // would mean the fixture store is missing a resource it claims to hold, + // and the verdict under test would be about a different store. + t.Fatalf("seed store resources: %d kind(s) refused: %+v", len(skipped), skipped) + } } // storedResourceCount reads the count validate reports out of its summary line. diff --git a/internal/app/archied/control_plane_validation_test.go b/internal/app/archied/control_plane_validation_test.go index 1a28fbf4a..856a172e5 100644 --- a/internal/app/archied/control_plane_validation_test.go +++ b/internal/app/archied/control_plane_validation_test.go @@ -1,50 +1,59 @@ package archied import ( + "context" "errors" + "fmt" "log/slog" + "os" + "path/filepath" "testing" + "time" "github.com/samcharles93/archie-core/internal/app/controlplane" "github.com/samcharles93/archie-core/internal/infrastructure/configuration" ) -// TestLoadConfigAcceptsStaleDatabaseOwnedValues is archie-core-i3qm at the boot -// entry point. docs/prds/runtime-control-plane.md, "Bootstrap, migration, and -// recovery": after migration, settings in TOML are ignored and cannot block -// State Store startup. archie-state-store boots through loadConfig and then -// opens the store; it never layers a control-plane resource over the file -// document (state_store.go), so a stale value in a setting the control plane -// owns must not fail boot. Every value below is one a control-plane resource -// replaces wholesale. -func TestLoadConfigAcceptsStaleDatabaseOwnedValues(t *testing.T) { - tests := []struct { - name string - body string - }{ - { - name: "provider base_url carrying userinfo", - body: "[providers.openai]\nclass = \"openai\"\nbase_url = \"https://token@example.com/v1\"\n", - }, - { - name: "negative poll_interval", - body: "poll_interval = \"-5s\"\n", - }, - { - name: "unrecognised dispatch.trigger", - body: "[dispatch]\ntrigger = \"labels\"\n", - }, - { - name: "negative containers.volume_ttl", - body: "[containers]\nvolume_ttl = \"-1m\"\n", - }, - { - name: "malformed repos test_glob", - body: "[[repos]]\nowner = \"acme\"\nname = \"app\"\ntest_glob = \"[\"\n", - }, - } +// staleDatabaseOwnedSettings are TOML fragments, each carrying one value a +// control-plane resource is seeded from and replaces wholesale. They are the +// values docs/prds/runtime-control-plane.md, +// "Bootstrap, migration, and recovery" says must not block a process: "after +// migration, settings in TOML are ignored and cannot block State Store +// startup". +var staleDatabaseOwnedSettings = []struct { + name string + body string +}{ + { + name: "provider base_url carrying userinfo", + body: "[providers.openai]\nclass = \"openai\"\nbase_url = \"https://token@example.com/v1\"\n", + }, + { + name: "negative poll_interval", + body: "poll_interval = \"-5s\"\n", + }, + { + name: "unrecognised dispatch.trigger", + body: "[dispatch]\ntrigger = \"labels\"\n", + }, + { + name: "negative containers.volume_ttl", + body: "[containers]\nvolume_ttl = \"-1m\"\n", + }, + { + name: "malformed repos test_glob", + body: "[[repos]]\nowner = \"acme\"\nname = \"app\"\ntest_glob = \"[\"\n", + }, +} - for _, tt := range tests { +// TestLoadConfigAcceptsStaleDatabaseOwnedValues is the first of the two layers +// archie-state-store boots through. loadConfig resolves the file and is where +// the loader used to reject these values; it must not, because the setting +// belongs to the control plane. The second layer is the boot itself, covered by +// TestStateStoreBootsWithStaleDatabaseOwnedValues -- passing only this one is +// not the item's claim. +func TestLoadConfigAcceptsStaleDatabaseOwnedValues(t *testing.T) { + for _, tt := range staleDatabaseOwnedSettings { t.Run(tt.name, func(t *testing.T) { path := t.TempDir() + "/config.toml" writeConfig(t, path, "bot_user = \"widget\"\n"+tt.body) @@ -57,6 +66,63 @@ func TestLoadConfigAcceptsStaleDatabaseOwnedValues(t *testing.T) { } } +// TestStateStoreBootsWithStaleDatabaseOwnedValues is the process-level half of +// the item, and the layer that actually decides it: archie-state-store seeds +// every control-plane resource from the same file config on a fresh database +// (state_store.go, control.ImportConfig), and that seed is what runs the +// resource's own validator. A seed that does not validate is skipped and +// reported, not fatal -- the value stays file-owned, archied refuses to run with +// it, and the operator's fix in config.toml is what clears it. +// staleBootGrace is how long one boot in +// TestStateStoreBootsWithStaleDatabaseOwnedValues may run before the test +// cancels it. It can be short because the assertion does not depend on it: a +// refusal arrives as an error either before the cancel (first select) or after +// it (second select, where a non-Canceled error fails the test). The wait only +// gives the serving path a chance to prove it did not return immediately. +const staleBootGrace = 250 * time.Millisecond + +func TestStateStoreBootsWithStaleDatabaseOwnedValues(t *testing.T) { + for _, tt := range staleDatabaseOwnedSettings { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.toml") + body := fmt.Sprintf( + "bot_user = 'archie'\ndb_path = %q\n%s[forge]\ntype = 'github'\nhost = 'https://github.example.com'\n", + filepath.Join(dir, "state.db"), tt.body, + ) + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + + // Loopback, so no token is required. Cancellation is the assertion: + // a boot that refuses the config returns before it, and one that + // serves returns only when the context ends. Cancel first, then + // assert -- racing startup against a wall clock is a coin flip. + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + errCh := make(chan error, 1) + go func() { + errCh <- RunStateStore(ctx, StateStoreOptions{Config: path, Listen: "127.0.0.1:0"}) + }() + + select { + case err := <-errCh: + t.Fatalf("RunStateStore returned before cancellation, want it to serve: %v", err) + case <-time.After(staleBootGrace): + } + cancel() + select { + case err := <-errCh: + if err != nil && !errors.Is(err, context.Canceled) { + t.Fatalf("RunStateStore returned %v, want nil or context.Canceled", err) + } + case <-time.After(shutdownGrace): + t.Fatal("RunStateStore did not return within the shutdown grace after cancellation") + } + }) + } +} + // TestRuntimeConfigFailsClosedOnAnInvalidDatabaseValue is the other half of the // same change: the checks that no longer run at load run on the effective // document, at the one place database resources are layered in. A stored value diff --git a/internal/app/archied/state_store.go b/internal/app/archied/state_store.go index b1b919d35..3ba9e9cb5 100644 --- a/internal/app/archied/state_store.go +++ b/internal/app/archied/state_store.go @@ -106,10 +106,21 @@ func RunStateStore(ctx context.Context, options StateStoreOptions) error { if err != nil { return err } - versions, err := control.ImportConfig(ctx, b.cfg) + versions, skipped, err := control.ImportConfig(ctx, b.cfg) if err != nil { return fmt.Errorf("import control-plane resources: %w", err) } + for _, skip := range skipped { + // Reported, not fatal: a stale TOML value in a setting the database owns + // must not stop this process from starting + // (docs/prds/runtime-control-plane.md, "Bootstrap, migration, and + // recovery"), and the resource stays absent, so the file's value is + // still the one in effect. Whether it may RUN is decided where it is + // used: boot.runtimeConfig validates the effective document and refuses + // to start archied or the Gateway with it. + b.log.Error("control-plane resource not seeded; the file config's value stays in effect", + "kind", skip.Kind, "err", skip.Err) + } if err := migrateLegacySchedules(ctx, control, b.cfg.DBPath, versions[controlplane.SchedulesKind]); err != nil { return fmt.Errorf("migrate schedules: %w", err) } diff --git a/internal/app/controlplane/controlplane_test.go b/internal/app/controlplane/controlplane_test.go index 499ce2c61..fcbe17061 100644 --- a/internal/app/controlplane/controlplane_test.go +++ b/internal/app/controlplane/controlplane_test.go @@ -3,6 +3,7 @@ package controlplane import ( "context" "encoding/json" + "errors" "testing" "time" @@ -74,7 +75,7 @@ func TestImportConfigSeedsWorkflowDefinitionsWithoutOverwritingOverride(t *testi resources := store.OpenTest(t) defer resources.Close() server := testServer(t, resources) - if _, err := server.ImportConfig(t.Context(), config.Config{}); err != nil { + if _, _, err := server.ImportConfig(t.Context(), config.Config{}); err != nil { t.Fatal(err) } resource, err := resources.Resource(t.Context(), WorkflowDefinitionsKind) @@ -96,7 +97,7 @@ func TestImportConfigSeedsWorkflowDefinitionsWithoutOverwritingOverride(t *testi if _, err := resources.PutResource(t.Context(), store.ResourceWrite{Kind: WorkflowDefinitionsKind, Value: value, ExpectedVersion: resource.Version, Actor: "test", Source: "test", RequestID: "override"}); err != nil { t.Fatal(err) } - if _, err := server.ImportConfig(t.Context(), config.Config{}); err != nil { + if _, _, err := server.ImportConfig(t.Context(), config.Config{}); err != nil { t.Fatal(err) } got, err := resources.Resource(t.Context(), WorkflowDefinitionsKind) @@ -114,7 +115,7 @@ func TestImportConfigSeedsPersonasWithoutOverwritingEdits(t *testing.T) { resources := store.OpenTest(t) defer resources.Close() server := testServer(t, resources) - if _, err := server.ImportConfig(t.Context(), config.Config{}); err != nil { + if _, _, err := server.ImportConfig(t.Context(), config.Config{}); err != nil { t.Fatal(err) } resource, err := resources.Resource(t.Context(), PersonasKind) @@ -133,7 +134,7 @@ func TestImportConfigSeedsPersonasWithoutOverwritingEdits(t *testing.T) { if _, err := resources.PutResource(t.Context(), store.ResourceWrite{Kind: PersonasKind, Value: value, ExpectedVersion: resource.Version, Actor: "test", Source: "test", RequestID: "persona-edit"}); err != nil { t.Fatal(err) } - if _, err := server.ImportConfig(t.Context(), config.Config{}); err != nil { + if _, _, err := server.ImportConfig(t.Context(), config.Config{}); err != nil { t.Fatal(err) } got, err := resources.Resource(t.Context(), PersonasKind) @@ -178,9 +179,15 @@ func TestRegistryRoutesAndValidatesDefinitions(t *testing.T) { resources := store.OpenTest(t) defer resources.Close() server := testServer(t, resources) - if _, err := server.ImportConfig(t.Context(), config.Config{Containers: config.ContainerConfig{PullPolicy: "missing"}}); err != nil { + // The image is required, here as in the file document: a seed without one is + // skipped rather than stored, and the Command below would then fail on the + // missing resource instead of on the pull policy it exists to reject. + if _, _, err := server.ImportConfig(t.Context(), config.Config{Containers: config.ContainerConfig{Image: "archie-agent:test", PullPolicy: "missing"}}); err != nil { t.Fatal(err) } + if _, err := resources.Resource(t.Context(), ContainerRuntimePoliciesKind); err != nil { + t.Fatalf("container runtime policies were not seeded: %v", err) + } catalog, err := server.Catalog(t.Context(), &pb.CatalogRequest{}) if err != nil { @@ -209,7 +216,7 @@ func TestProviderSeedKeepsReferencesAndNeverResolvedSecrets(t *testing.T) { defer resources.Close() server := testServer(t, resources) cfg := config.Config{Providers: map[string]config.Provider{"openai": {Class: "openai", APIKeyEnv: "OPENAI_API_KEY", APIKey: config.SecretRef{Engine: "env", Key: "OPENAI_API_KEY"}}}} - if _, err := server.ImportConfig(context.Background(), cfg); err != nil { + if _, _, err := server.ImportConfig(context.Background(), cfg); err != nil { t.Fatal(err) } resource, err := resources.Resource(t.Context(), ProviderSettingsKind) @@ -235,7 +242,7 @@ func TestHistoryCarriesEveryRevisionWithItsAudit(t *testing.T) { resources := store.OpenTest(t) defer resources.Close() server := testServer(t, resources) - if _, err := server.ImportConfig(t.Context(), config.Config{}); err != nil { + if _, _, err := server.ImportConfig(t.Context(), config.Config{}); err != nil { t.Fatal(err) } seeded, err := server.Query(t.Context(), &pb.QueryRequest{Kind: WorkflowExecutionSettingsKind}) @@ -269,3 +276,43 @@ func TestHistoryCarriesEveryRevisionWithItsAudit(t *testing.T) { t.Fatalf("history for an unknown kind = %v, want NotFound", err) } } + +// TestImportConfigSkipsASeedItCannotValidate is the reason a bad seed is not +// fatal. docs/prds/runtime-control-plane.md, "Bootstrap, migration, and +// recovery": after migration, settings in TOML are ignored and cannot block +// State Store startup -- and ImportConfig is on that startup path. The value is +// not stored either, so the setting stays file-owned: the file's value is still +// the one in effect, and editing config.toml is still the fix. Writing the +// invalid value instead would hand ownership to the database and leave the +// operator with a file edit that does nothing. +func TestImportConfigSkipsASeedItCannotValidate(t *testing.T) { + t.Parallel() + + resources := store.OpenTest(t) + defer resources.Close() + server := testServer(t, resources) + // Every other field is filled in the way the loader fills it (Validate + // documents that it does not apply defaults), so the malformed glob is the + // only seed that can be refused. + cfg := validConfigForValidation() + cfg.Repos = []config.Repo{{Owner: "acme", Name: "app", TestGlob: "["}} + + versions, skipped, err := server.ImportConfig(t.Context(), cfg) + if err != nil { + t.Fatalf("ImportConfig: %v, want a skipped seed rather than a fatal one", err) + } + if _, stored := versions[RepositoryPoliciesKind]; stored { + t.Error("the resource was seeded, want the invalid seed skipped") + } + if len(skipped) != 1 || skipped[0].Kind != RepositoryPoliciesKind || skipped[0].Err == nil { + t.Fatalf("skipped = %+v, want the repository policies and the reason", skipped) + } + if _, err := resources.Resource(t.Context(), RepositoryPoliciesKind); !errors.Is(err, store.ErrResourceNotFound) { + t.Errorf("repository policies Resource = %v, want ErrResourceNotFound: an invalid seed must not be stored", err) + } + // Kinds that are fine are still seeded: one skipped kind does not abandon + // the rest of the migration. + if _, err := resources.Resource(t.Context(), WorkflowDefinitionsKind); err != nil { + t.Errorf("workflow definitions: %v, want them seeded alongside the skipped kind", err) + } +} diff --git a/internal/app/controlplane/operational_settings.go b/internal/app/controlplane/operational_settings.go index 1d33d197e..03b9d4591 100644 --- a/internal/app/controlplane/operational_settings.go +++ b/internal/app/controlplane/operational_settings.go @@ -6,6 +6,7 @@ import ( "time" "github.com/samcharles93/archie-core/internal/config" + "github.com/samcharles93/archie-core/internal/infrastructure/configuration" ) const ( @@ -56,6 +57,14 @@ func validateRepositories(input []byte) error { return fmt.Errorf("duplicate repository %q", repo.FullName()) } seen[repo.FullName()] = struct{}{} + // The repository-policies resource replaces the file's list, so the + // rules that judge the file's list have to hold here too. They come + // from the configuration package rather than being restated: a + // resource that accepts what configuration.Validate rejects becomes a + // stored value the operator cannot clear by editing config.toml. + if err := configuration.ValidateTestGlob(repo.ResolvedTestGlob()); err != nil { + return err + } } return nil }) @@ -72,6 +81,12 @@ func validateScheduling(input []byte) error { return fmt.Errorf("poll_interval must be a non-negative duration") } } + // Same reason as the repository glob above: RuntimeConfig replaces + // cfg.Dispatch from this resource, and the daemon refuses to start with a + // trigger it does not know. + if !configuration.DispatchTriggerValid(policy.Dispatch.Trigger) { + return fmt.Errorf("dispatch.trigger %q is not a discovery rule the daemon can poll with", policy.Dispatch.Trigger) + } return nil }) } @@ -84,6 +99,11 @@ func validateContainers(input []byte) error { if settings.PullPolicy != "" && settings.PullPolicy != "missing" && settings.PullPolicy != "always" { return fmt.Errorf("pull_policy must be missing or always") } + // The image is required for autonomous workflow workers; the same rule + // holds for the stored policies that replace the file's [containers]. + if strings.TrimSpace(settings.Image) == "" { + return fmt.Errorf("container image is required for autonomous workflow workers") + } return nil }) } diff --git a/internal/app/controlplane/registry.go b/internal/app/controlplane/registry.go index 629421c93..b7ecf168a 100644 --- a/internal/app/controlplane/registry.go +++ b/internal/app/controlplane/registry.go @@ -61,8 +61,28 @@ func (d Definition) Decode(input []byte) ([]byte, error) { return encoded, nil } -func (s *Server) ImportConfig(ctx context.Context, cfg config.Config) (map[string]int64, error) { +// SeedSkip records a resource ImportConfig could not seed from the file config, +// and why. It is reported rather than returned as an error: see ImportConfig. +type SeedSkip struct { + Kind string + Err error +} + +// ImportConfig seeds every control-plane resource that has no stored value yet +// from the file config, and reports the version of every kind it left alone. +// +// A seed the resource's own validator refuses is SKIPPED and returned in skipped, +// not fatal. The setting it would have carried stays file-owned, so the value in +// effect is still the file's, and the fix is still the file's. +// docs/prds/runtime-control-plane.md, "Bootstrap, migration, and recovery": +// after migration, settings in TOML are ignored and cannot block State Store +// startup -- and this import runs on the State Store's startup path. Fail closed +// belongs to the process that uses the value: boot.runtimeConfig validates the +// effective document and refuses to start archied with a value it cannot run +// with, which is what keeps an invalid value from taking effect silently. +func (s *Server) ImportConfig(ctx context.Context, cfg config.Config) (map[string]int64, []SeedSkip, error) { versions := make(map[string]int64, len(s.ordered)) + var skipped []SeedSkip for _, definition := range s.ordered { resource, err := s.store.Resource(ctx, definition.Kind) if err == nil { @@ -70,19 +90,27 @@ func (s *Server) ImportConfig(ctx context.Context, cfg config.Config) (map[strin continue } if !errors.Is(err, store.ErrResourceNotFound) { - return nil, err + return nil, nil, err } value, err := definition.seededValue(cfg) if err != nil { - return nil, err + // Skipped and reported, not fatal: nothing is written, so the kind + // stays absent and the file document's value is the one in effect, + // while the process that would RUN the value still fails closed on + // it (boot.runtimeConfig validates the effective document). + // seededValue wraps a seed's encode and validation failures together, + // which is why both take this path: the alternative is a second seed + // implementation here to tell them apart. + skipped = append(skipped, SeedSkip{Kind: definition.Kind, Err: err}) + continue } resource, err = s.store.PutResource(ctx, store.ResourceWrite{Kind: definition.Kind, Value: value, Actor: "system:migration", Source: "legacy-config", RequestID: "import:" + definition.Kind, ExpectedVersion: 0, At: time.Now().UTC()}) if err != nil { - return nil, fmt.Errorf("seed %s: %w", definition.Kind, err) + return nil, nil, fmt.Errorf("seed %s: %w", definition.Kind, err) } versions[definition.Kind] = resource.Version } - return versions, nil + return versions, skipped, nil } // seededValue is the value the State Store writes for a kind it does not hold diff --git a/internal/app/controlplane/validators_agree_test.go b/internal/app/controlplane/validators_agree_test.go new file mode 100644 index 000000000..814a058ed --- /dev/null +++ b/internal/app/controlplane/validators_agree_test.go @@ -0,0 +1,150 @@ +package controlplane + +import ( + "encoding/json" + "testing" + "time" + + "github.com/samcharles93/archie-core/internal/config" + "github.com/samcharles93/archie-core/internal/infrastructure/configuration" +) + +// TestResourceValidatorsRejectWhatEffectiveValidationRejects ties the two rule +// sets that guard the same fields. A control-plane resource is seeded from the +// file document and then replaces it wholesale (Client.RuntimeConfig), so a +// value the file path accepts must not be storable through the resource, and a +// value the resource accepts must not stop the daemon's validation of the +// effective document. Guards that live only on one side are how a stale TOML +// value becomes a database value the operator cannot fix by editing config.toml. +// +// Each case mutates the same field the resource carries, so the two halves +// cannot drift apart without failing here. +func TestResourceValidatorsRejectWhatEffectiveValidationRejects(t *testing.T) { + tests := []struct { + name string + kind string + value any + mutate func(cfg *config.Config) + wantErr bool + }{ + { + name: "valid scheduling policy", + kind: SchedulingPolicyKind, + value: map[string]any{"poll_interval": "1m0s", "max_retries": 0, "dispatch": map[string]any{"trigger": "assignee"}}, + mutate: func(cfg *config.Config) {}, + wantErr: false, + }, + { + name: "unrecognised dispatch trigger", + kind: SchedulingPolicyKind, + value: map[string]any{"poll_interval": "1m0s", "max_retries": 0, "dispatch": map[string]any{"trigger": "labels"}}, + mutate: func(cfg *config.Config) { cfg.Dispatch.Trigger = "labels" }, + wantErr: true, + }, + { + name: "negative poll interval", + kind: SchedulingPolicyKind, + value: map[string]any{"poll_interval": "-5s", "max_retries": 0, "dispatch": map[string]any{"trigger": "assignee"}}, + mutate: func(cfg *config.Config) { cfg.PollInterval = config.Duration(-5 * time.Second) }, + wantErr: true, + }, + { + name: "valid repository policies", + kind: RepositoryPoliciesKind, + value: []config.Repo{{Owner: "acme", Name: "app", TestGlob: "**/*_test.go"}}, + mutate: func(cfg *config.Config) {}, + wantErr: false, + }, + { + name: "malformed test glob", + kind: RepositoryPoliciesKind, + value: []config.Repo{{Owner: "acme", Name: "app", TestGlob: "["}}, + mutate: func(cfg *config.Config) { cfg.Repos = []config.Repo{{Owner: "acme", Name: "app", TestGlob: "["}} }, + wantErr: true, + }, + { + name: "valid container runtime policies", + kind: ContainerRuntimePoliciesKind, + value: config.ContainerConfig{Image: "archie-agent:test", PullPolicy: "missing"}, + mutate: func(cfg *config.Config) {}, + wantErr: false, + }, + { + name: "container runtime policies without an image", + kind: ContainerRuntimePoliciesKind, + value: config.ContainerConfig{PullPolicy: "missing"}, + mutate: func(cfg *config.Config) { cfg.Containers.Image = "" }, + wantErr: true, + }, + { + name: "negative container volume ttl", + kind: ContainerRuntimePoliciesKind, + value: config.ContainerConfig{Image: "archie-agent:test", VolumeTTL: config.Duration(-time.Minute)}, + mutate: func(cfg *config.Config) { cfg.Containers.VolumeTTL = config.Duration(-time.Minute) }, + wantErr: true, + }, + { + name: "provider base URL carrying userinfo", + kind: ProviderSettingsKind, + value: map[string]any{"acme": map[string]any{"class": "openai", "base_url": "https://user:pass@example.com"}}, + mutate: func(cfg *config.Config) { + cfg.Providers = map[string]config.Provider{"acme": {Class: "openai", BaseURL: "https://user:pass@example.com"}} + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := validConfigForValidation() + tt.mutate(&cfg) + configErr := configuration.Validate(&cfg) + + encoded, err := json.Marshal(tt.value) + if err != nil { + t.Fatal(err) + } + resourceErr := decodeResource(t, tt.kind, encoded) + + if !tt.wantErr { + if configErr != nil { + t.Errorf("configuration.Validate = %v, want nil for a valid value", configErr) + } + if resourceErr != nil { + t.Errorf("%s Decode = %v, want nil for a valid value", tt.kind, resourceErr) + } + return + } + if configErr == nil { + t.Errorf("configuration.Validate accepted the value, want the effective document to reject it") + } + if resourceErr == nil { + t.Errorf("%s accepted the value, want the resource validator to reject what the effective document rejects", tt.kind) + } + }) + } +} + +// decodeResource runs the resource validator exactly as ImportConfig does when +// it seeds a kind from the file config (registry.go), against the production +// server the composition roots build (testServer). +func decodeResource(t *testing.T, kind string, encoded []byte) error { + t.Helper() + _, err := testServer(t, nil).definitions[kind].Decode(encoded) + return err +} + +// validConfigForValidation is a document every check passes, so each case above +// fails only for the value it changes -- same fixture the configuration package +// uses, restated here because Validate documents that it does not apply +// defaults. +func validConfigForValidation() config.Config { + return config.Config{ + BotUser: "archie-bot", + PollInterval: config.Duration(60 * time.Second), + Dispatch: config.Dispatch{Trigger: "assignee"}, + Forge: config.Forge{Type: "none"}, + Containers: config.ContainerConfig{Image: "ghcr.io/samcharles93/archie-agent:latest"}, + Services: config.Services{config.ServiceNameGateway: {Target: "127.0.0.1:8585"}}, + } +} diff --git a/internal/infrastructure/configuration/apply.go b/internal/infrastructure/configuration/apply.go index fa39ffe49..d7d900151 100644 --- a/internal/infrastructure/configuration/apply.go +++ b/internal/infrastructure/configuration/apply.go @@ -30,23 +30,30 @@ func ApplyOverlayValues(cfg *config.Config, overrides map[string]any) error { return applyOverlayMapping(cfg, overrides) } -// applyOverlayFile layers the overlay file at path over cfg, with the same -// field-level precedence ApplyOverlayValues gives a map overlay. The file -// overlay path needs it for the reason recorded on foldOverrides: decoding an -// overlay into cfg replaces a map-valued entry wholesale, so a file naming -// only services.state.target cleared the target_token the base file set. Both -// overlay paths go through the fold, so neither can drift from the other. -func applyOverlayFile(path string, cfg *config.Config) error { - doc, err := decodeFileMapping(path) +// applyOverlayFile layers the overlay file at path over target, with the same +// field-level precedence ApplyOverlayValues gives a map overlay, and reports the +// file's keys that nothing consumes. target is whatever the overlay addresses: +// the whole config for a main file, one sub-struct for a feature file. +// +// Both overlay forms go through it -- the single overlay file, and every file an +// overlay directory contributes -- so neither form can lose the fields of a +// map-valued entry it only partly addresses. Decoding an overlay into target +// directly replaces such an entry wholesale, which is how an overlay naming only +// services.state.target cleared the target_token the base file set. +func applyOverlayFile(path string, target any) ([]string, error) { + mapping, err := decodeFileMapping(path) if err != nil { - return err + return nil, err } - return applyOverlayMapping(cfg, doc) + if err := applyOverlayMapping(target, mapping); err != nil { + return nil, err + } + return overlayFileKeys(path, mapping, structTypeOf(target)) } // applyOverlayMapping is the decode both overlay paths share: the fields each // map-valued entry the overlay does not name are folded forward out of the -// entry cfg already holds, and the completed mapping is then decoded over cfg. +// entry target already holds, and the completed mapping is then decoded over it. // // yaml gives a struct field the field-level treatment ApplyOverlayValues // promises, but a map VALUE is decoded into a fresh zero value: an entry the @@ -55,8 +62,8 @@ func applyOverlayFile(path string, cfg *config.Config) error { // target_token. Folding here, once, is what makes the promise hold for every // map of structs (services, image.hosted, image.local, providers) instead of // at each field's reader. -func applyOverlayMapping(cfg *config.Config, doc map[string]any) error { - folded, err := foldOverrides(reflect.ValueOf(cfg), doc) +func applyOverlayMapping(target any, doc map[string]any) error { + folded, err := foldOverrides(reflect.ValueOf(target), doc) if err != nil { return err } @@ -64,12 +71,22 @@ func applyOverlayMapping(cfg *config.Config, doc map[string]any) error { if err != nil { return fmt.Errorf("%w: encoding config overlay: %w", ErrUnreadable, err) } - if err := yaml.Unmarshal(data, cfg); err != nil { + if err := yaml.Unmarshal(data, target); err != nil { return fmt.Errorf("%w: parsing config overlay: %w", ErrUnreadable, err) } return nil } +// structTypeOf returns the struct type target points at, so the fold and the key +// report can inspect it without an instance. +func structTypeOf(target any) reflect.Type { + t := reflect.TypeOf(target) + for t != nil && t.Kind() == reflect.Pointer { + t = t.Elem() + } + return t +} + // foldOverrides returns doc with the fields its map entries omit carried // forward from the entries cfg already holds. It descends through structs, so // {"image": {"hosted": {...}}} folds the same way as an override written at the @@ -182,7 +199,16 @@ func mergeMapping(base, override map[string]any) map[string]any { // yamlField returns the field of struct v that the decode reads key into. The // yaml tag is the one consulted because the decode below is yaml's. func yamlField(v reflect.Value, key string) (reflect.Value, bool) { - t := v.Type() + field, ok := yamlFieldOf(v.Type(), key) + if !ok { + return reflect.Value{}, false + } + return v.FieldByIndex(field.Index), true +} + +// yamlFieldOf is yamlField without a value: the overlay key report walks a raw +// mapping against the target's type before anything is decoded into it. +func yamlFieldOf(t reflect.Type, key string) (reflect.StructField, bool) { for i := range t.NumField() { field := t.Field(i) if field.PkgPath != "" { @@ -196,8 +222,8 @@ func yamlField(v reflect.Value, key string) (reflect.Value, bool) { name = strings.ToLower(field.Name) } if name == key { - return v.Field(i), true + return field, true } } - return reflect.Value{}, false + return reflect.StructField{}, false } diff --git a/internal/infrastructure/configuration/decode.go b/internal/infrastructure/configuration/decode.go index 08975ef44..585234efe 100644 --- a/internal/infrastructure/configuration/decode.go +++ b/internal/infrastructure/configuration/decode.go @@ -31,7 +31,7 @@ func decodeConfigFileKeys(path string, target any) ([]string, error) { // carries, with no typed target. An overlay needs this shape: a typed decode // cannot distinguish a key the file omits from a key it sets to the zero // value, and that distinction is what carrying the fields of a partially -// addressed map entry forward is built on (see applyFileOverlay). +// addressed map entry forward is built on (see applyOverlayFile). func decodeFileMapping(path string) (map[string]any, error) { var doc map[string]any switch filepath.Ext(path) { @@ -98,15 +98,25 @@ func decodeYAML(path string, target any) error { // into cfg directly. Memory and tools are sub-structs whose keys sit at the // top level of their own file rather than nested under a "memory:" or // "tools:" key, so they decode into the sub-struct. -func decodeFeature(cfg *config.Config, feature Feature, path string) error { +// +// An overlay layer's feature file is folded over the base layer, for the same +// reason and with the same precedence as a main overlay file. The keys that fold +// reports are dropped here: only a main file reports unknown keys today +// (decodeMain), and a feature file reporting a second, differently-shaped set +// would be a new surface rather than a fix to this one. +func decodeFeature(cfg *config.Config, feature Feature, path string, layer Layer) error { + target := any(cfg) switch feature { case FeatureMemory: - return decodeYAML(path, &cfg.Memory) + target = &cfg.Memory case FeatureTools: - return decodeYAML(path, &cfg.Tools) - default: - return decodeYAML(path, cfg) + target = &cfg.Tools } + if layer == LayerOverlay { + _, err := applyOverlayFile(path, target) + return err + } + return decodeYAML(path, target) } // decodeExtra decodes an unrecognised conf.d/ file into cfg.Extra under name. diff --git a/internal/infrastructure/configuration/loader.go b/internal/infrastructure/configuration/loader.go index beb64085a..5cff2a138 100644 --- a/internal/infrastructure/configuration/loader.go +++ b/internal/infrastructure/configuration/loader.go @@ -5,6 +5,7 @@ import ( "log/slog" "os" "path/filepath" + "reflect" "sort" "github.com/samcharles93/archie-core/internal/config" @@ -175,11 +176,8 @@ func (l *Loader) overlayFile(basePath, overlayPath string) (*Document, error) { if overlayPath != "" { // The overlay is folded over the base config rather than decoded into // it: a decode replaces a map-valued entry wholesale, clearing the - // fields of that entry the overlay does not name (applyFileOverlay). - if err := applyOverlayFile(overlayPath, &doc.Config); err != nil { - return nil, err - } - cfgKeys, err = overlayFileKeys(overlayPath) + // fields of that entry the overlay does not name (applyOverlayFile). + cfgKeys, err = applyOverlayFile(overlayPath, &doc.Config) if err != nil { return nil, err } @@ -194,13 +192,69 @@ func (l *Loader) overlayFile(basePath, overlayPath string) (*Document, error) { return l.finalize(doc, validateBootstrap) } -// overlayFileKeys reports the top-level keys of an overlay file that the -// config decode does not consume. The file itself reaches the config through -// applyOverlayFile, which needs its raw mapping; this typed decode exists only -// for the key report, which only TOML's decoder can produce. -func overlayFileKeys(path string) ([]string, error) { - var scratch config.Config - return decodeConfigFileKeys(path, &scratch) +// overlayFileKeys reports the keys of an overlay file that nothing consumes. It +// has two sources, because the file's own decode and the fold's apply decode do +// not match keys the same way: TOML accepts a key that differs only in case, +// while the yaml decode the fold uses does not. A key only the file's decode +// matches would therefore be consumed by the report below and dropped in +// silence by the apply, so the apply's own view of the mapping is unioned in. +// +// The second decode here is deliberate: it is what carries TOML's Undecoded() +// view of the file, which the raw mapping cannot report (it consumes every key). +func overlayFileKeys(path string, mapping map[string]any, target reflect.Type) ([]string, error) { + decoded, err := decodeConfigFileKeys(path, reflect.New(target).Interface()) + if err != nil { + return nil, err + } + return append(decoded, unmatchableKeys(mapping, target)...), nil +} + +// unmatchableKeys returns the dotted paths of the keys in mapping that the fold's +// yaml decode cannot match to a field of target. It descends the way foldOverrides +// does -- through structs and through string-keyed maps of structs -- so it sees +// the same keys the apply does. A slice of tables is not descended: the yaml +// decode replaces it wholesale, so anything inside it is a value, not a key. +func unmatchableKeys(mapping map[string]any, target reflect.Type) []string { + var out []string + for key, value := range mapping { + field, ok := yamlFieldOf(target, key) + if !ok { + out = append(out, key) + continue + } + sub, ok := value.(map[string]any) + if !ok { + continue + } + switch field.Type.Kind() { + case reflect.Struct: + out = append(out, prefixed(key, unmatchableKeys(sub, field.Type))...) + case reflect.Map: + entry := field.Type.Elem() + for entry.Kind() == reflect.Pointer { + entry = entry.Elem() + } + if entry.Kind() != reflect.Struct || field.Type.Key().Kind() != reflect.String { + continue + } + for entryName, entryValue := range sub { + entryMapping, ok := entryValue.(map[string]any) + if !ok { + continue + } + out = append(out, prefixed(key+"."+entryName, unmatchableKeys(entryMapping, entry))...) + } + } + } + return out +} + +// prefixed qualifies a nested key report with the path that reached it. +func prefixed(prefix string, keys []string) []string { + for i, key := range keys { + keys[i] = prefix + "." + key + } + return keys } // unknownKeys returns the keys present in both a and b: one config file @@ -303,7 +357,7 @@ func (l *Loader) loadDir(doc *Document, dir string, layer Layer) error { switch path, isYAMLFile, ok := files.main(); { case ok: - if err := l.decodeMain(doc, path, isYAMLFile); err != nil { + if err := l.decodeMain(doc, path, isYAMLFile, layer); err != nil { return err } doc.Provenance.record(Origin{Path: path, Role: RoleMain, Layer: layer}) @@ -313,7 +367,7 @@ func (l *Loader) loadDir(doc *Document, dir string, layer Layer) error { for _, feature := range files.sortedFeatures() { path := files.features[feature] - if err := decodeFeature(&doc.Config, feature, path); err != nil { + if err := decodeFeature(&doc.Config, feature, path, layer); err != nil { return err } doc.Provenance.record(Origin{Path: path, Role: RoleFeature, Layer: layer, Feature: feature}) @@ -330,8 +384,20 @@ func (l *Loader) loadDir(doc *Document, dir string, layer Layer) error { return nil } -// decodeMain decodes the daemon-level file in whichever format it uses. -func (l *Loader) decodeMain(doc *Document, path string, isYAMLFile bool) error { +// decodeMain decodes the daemon-level file in whichever format it uses. An +// overlay layer's file is FOLDED over what the base layer decoded, the same way +// a single overlay file folds over its base -- otherwise a map-valued entry it +// only partly addresses would clear the fields it does not name, which is the +// archie-core-e2e2 failure on the directory form of the same flag. +func (l *Loader) decodeMain(doc *Document, path string, isYAMLFile bool, layer Layer) error { + if layer == LayerOverlay { + cfgKeys, err := applyOverlayFile(path, &doc.Config) + if err != nil { + return err + } + doc.UnknownKeys = append(doc.UnknownKeys, cfgKeys...) + return decodeSchedulingFile(path, &doc.Scheduling) + } if isYAMLFile { if err := decodeYAML(path, &doc.Config); err != nil { return err diff --git a/internal/infrastructure/configuration/loader_test.go b/internal/infrastructure/configuration/loader_test.go index b8c1a7bdc..f61c14da3 100644 --- a/internal/infrastructure/configuration/loader_test.go +++ b/internal/infrastructure/configuration/loader_test.go @@ -241,10 +241,10 @@ func TestLoadTelegramTokenSecretRefAndLegacyFallback(t *testing.T) { // settings the control plane owns are layered over it later; a stale TOML value // in one of those must not fail the load (docs/prds/runtime-control-plane.md, // "Bootstrap, migration, and recovery"). So a check over a database-owned -// setting is asserted on the effective document instead -- the same document -// boot.runtimeConfig validates after the merge. Each case asserts both halves -// it applies to, so a check that quietly disappeared fails the case rather than -// passing it. +// setting is asserted with Validate instead -- what boot.runtimeConfig runs over +// the document a process will use, and the checks the loader no longer applies. +// Each case asserts both halves it applies to, so a check that quietly +// disappeared fails the case rather than passing it. func TestLoadRejectsInvalidConfigEnumsAndGlobs(t *testing.T) { tests := []struct { name string @@ -309,7 +309,7 @@ func TestLoadRejectsInvalidConfigEnumsAndGlobs(t *testing.T) { t.Fatalf("File: %v (a database-owned value must not fail the load)", err) } if err := Validate(&doc.Config); err == nil { - t.Fatal("Validate(effective document) = nil, want the value rejected there") + t.Fatal("Validate = nil, want the value rejected there") } }) } @@ -537,11 +537,10 @@ func TestLoadContainerVolumeTTL(t *testing.T) { } // TestLoadRejectsNegativeContainerVolumeTTL: containers are a control-plane -// resource (container-runtime-policies), so a negative volume_ttl is judged on -// the effective document rather than on the file the load path reads -- a stale -// TOML value in a database-owned setting must not fail a process's startup. The -// second half of the assertion is what keeps dropping the check from passing -// this test. +// resource (container-runtime-policies), so a negative volume_ttl is judged by +// Validate rather than by the load path -- a stale TOML value in a database-owned +// setting must not fail a process's startup. The second half of the assertion is +// what keeps dropping the check from passing this test. func TestLoadRejectsNegativeContainerVolumeTTL(t *testing.T) { path := filepath.Join(t.TempDir(), "config.toml") contents := "bot_user = \"widget\"\n" + @@ -557,7 +556,7 @@ func TestLoadRejectsNegativeContainerVolumeTTL(t *testing.T) { t.Fatalf("File: %v (a database-owned value must not fail the load)", err) } if err := Validate(&doc.Config); err == nil { - t.Fatal("Validate(effective document) accepted a negative containers.volume_ttl") + t.Fatal("Validate accepted a negative containers.volume_ttl") } } @@ -635,15 +634,16 @@ func TestOverlay(t *testing.T) { // (TestApplyOverlayValuesPreservesOmittedNestedFields). // // It goes through Loader.Resolve, the production entry point, because the -// hazard is a property of how the overlay file is applied, not of any one -// decode helper: Loader.overlayFile decodes the overlay into the very config -// the base file produced, and a map-valued entry is replaced wholesale, so -// every field of that entry the overlay does not name is cleared. +// hazard was a property of how the overlay file is applied, not of any one +// decode helper: Loader.overlayFile decoded the overlay into the very config +// the base file produced, and a map-valued entry is replaced wholesale, so every +// field of that entry the overlay does not name was cleared. // // That is archie-core-e2e2 on a user-visible path. deployments/dev.toml is the // documented -config-overlay argument and sets [services.state] target/listen, // while the installed config.toml is where [services.state].target_token lives: -// the overlay silently drops the token. +// the overlay silently dropped the token. The directory form of the same flag is +// covered by TestDirOverlayPreservesOmittedMapEntryFields. func TestResolveFileOverlayPreservesOmittedMapEntryFields(t *testing.T) { tests := []struct { name string @@ -757,18 +757,21 @@ func TestResolveFileOverlayPreservesOmittedMapEntryFields(t *testing.T) { // TestLoadAcceptsStaleDatabaseOwnedValues is the bootstrap half of the layer // split (archie-core-i3qm). docs/prds/runtime-control-plane.md, // "Bootstrap, migration, and recovery": after migration, settings in TOML are -// ignored and cannot block State Store startup. archie-state-store never layers -// a database resource over its file document -- state_store.go:78 resolves the -// file and goes straight to opening the store -- so a stale value in a setting -// the control plane owns must not fail the load. +// ignored and cannot block State Store startup. The other half is the boot +// itself -- archie-state-store seeds these same settings into the control plane +// on a fresh database, and a seed it cannot validate is skipped rather than fatal +// (TestStateStoreBootsWithStaleDatabaseOwnedValues boots the process for that). // // Each case is a setting with a control-plane resource behind it, seeded from // that field by controlplane.Server.ImportConfig: providers // (provider-settings), poll_interval and dispatch (scheduling-policy), // containers (container-runtime-policies), repositories -// (repository-policies). The second half of the assertion is what keeps this -// from being a weaker gate: the effective document, judged after the control -// plane has layered its own value in, must still reject the value. +// (repository-policies). The second assertion is what keeps this from being a +// weaker gate: the value is still rejected by Validate, which is what the daemon +// and the Gateway run over the document a process will actually use. Nothing in +// this package layers that document, so the check here is the file document on +// its own -- the test is that the loader judges no database-owned field, not +// that the value is harmless. func TestLoadAcceptsStaleDatabaseOwnedValues(t *testing.T) { tests := []struct { name string @@ -808,7 +811,7 @@ func TestLoadAcceptsStaleDatabaseOwnedValues(t *testing.T) { t.Fatalf("File: %v (a stale database-owned value must not fail the load)", err) } if err := Validate(&doc.Config); err == nil { - t.Fatal("Validate(effective document) = nil, want the control plane's own layer to reject it") + t.Fatal("Validate = nil, want the control plane's own layer to reject it") } }) } @@ -818,6 +821,82 @@ func TestLoadAcceptsStaleDatabaseOwnedValues(t *testing.T) { // so each case above adds one stale setting and nothing else. const fileConfigPrefix = "bot_user = \"widget\"\nwork_dir = \"/base/work\"\n" +// TestDirOverlayPreservesOmittedMapEntryFields is the directory form of the +// same guarantee TestResolveFileOverlayPreservesOmittedMapEntryFields pins. +// -config-overlay accepts a directory as well as a file, Loader.Dir documents +// "the same field-level precedence as [Loader.Overlay]", and Loader.Resolve +// routes a directory source to Dir -- so a map-valued entry the overlay only +// partly addresses must keep the fields it does not name on both forms of the +// flag, including when the entry arrives through a feature file. +func TestDirOverlayPreservesOmittedMapEntryFields(t *testing.T) { + tests := []struct { + name string + baseFiles map[string]string + overlayFiles map[string]string + check func(t *testing.T, cfg config.Config) + }{ + { + name: "main config keeps the token it does not name", + baseFiles: map[string]string{ + "config.yaml": "bot_user: widget\nservices:\n state:\n target: 127.0.0.1:9090\n target_token: secret\n", + }, + overlayFiles: map[string]string{ + "config.yaml": "services:\n state:\n target: 10.0.0.5:9090\n", + }, + check: func(t *testing.T, cfg config.Config) { + t.Helper() + got := cfg.Services[config.ServiceNameState] + if got.Target != "10.0.0.5:9090" { + t.Errorf("services.state.target = %q, want the overlay's 10.0.0.5:9090", got.Target) + } + if got.TargetToken != "secret" { + t.Errorf("services.state.target_token = %q, want the base's secret: a directory overlay must not clear a field it does not name", got.TargetToken) + } + }, + }, + { + name: "feature file keeps the fields it does not name", + baseFiles: map[string]string{ + "config.yaml": "bot_user: widget\n", + "config.models.yaml": "providers:\n anthropic:\n class: anthropic\n api_key_env: ANTHROPIC_API_KEY\n", + }, + overlayFiles: map[string]string{ + "config.models.yaml": "providers:\n anthropic:\n base_url: https://proxy.example\n", + }, + check: func(t *testing.T, cfg config.Config) { + t.Helper() + got := cfg.Providers["anthropic"] + want := config.Provider{ + Class: "anthropic", + APIKeyEnv: "ANTHROPIC_API_KEY", + BaseURL: "https://proxy.example", + } + if got != want { + t.Errorf("providers.anthropic = %+v, want %+v", got, want) + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + baseDir, overlayDir := tmpConfigDir(t), tmpConfigDir(t) + for name, content := range tt.baseFiles { + writeFile(t, baseDir, name, content) + } + for name, content := range tt.overlayFiles { + writeFile(t, overlayDir, name, content) + } + + doc, err := New(nil).Resolve(baseDir, overlayDir) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + tt.check(t, doc.Config) + }) + } +} + func TestLoadBytesDoesNotExist(t *testing.T) { _, err := loadBytes(nil) if err == nil { diff --git a/internal/infrastructure/configuration/unknown_keys_test.go b/internal/infrastructure/configuration/unknown_keys_test.go index 7896e664c..48a3b7d65 100644 --- a/internal/infrastructure/configuration/unknown_keys_test.go +++ b/internal/infrastructure/configuration/unknown_keys_test.go @@ -91,28 +91,55 @@ func TestUnknownSchedulingKeyIsStillReported(t *testing.T) { // TestOverlayUnknownKeysAreStillReported pins that the overlay path kept its // drift detection when the overlay file started being applied through the -// config fold (applyOverlayFile) instead of being decoded into the document's -// config: the key report comes from a separate decode, so losing it would be -// silent. +// config fold instead of being decoded into the document's config. That matters +// twice over there: the report comes from a separate decode, and a file is +// parsed by TOML while the fold applies it with a yaml decode, which do not +// match keys the same way. A key the APPLY cannot consume has to be reported, +// or a spelling that works in the base config would silently do nothing in the +// overlay. func TestOverlayUnknownKeysAreStillReported(t *testing.T) { - dir := t.TempDir() - basePath := filepath.Join(dir, "config.toml") - overlayPath := filepath.Join(dir, "dev.toml") - if err := os.WriteFile(basePath, []byte(minimalValidConfigTOML), 0o600); err != nil { - t.Fatal(err) - } - // typo: concurrancy. Unknown to both decode targets, so it is a real - // unknown key rather than Hazard 1's cross-target case. - if err := os.WriteFile(overlayPath, []byte("[containers]\nmax_concurrancy = 4\n"), 0o600); err != nil { - t.Fatal(err) + tests := []struct { + name string + overlay string + wantUnknown string + }{ + { + name: "a key no decode target consumes", + overlay: "[containers]\nmax_concurrancy = 4\n", // typo: concurrancy + wantUnknown: "containers.max_concurrancy", + }, + { + // TOML matches keys case-insensitively, so the file's own decode + // consumes BOT_USER and reports nothing; the apply folds the file's + // mapping with a yaml decode, which does not match it. Without the + // second source for this report the value would be dropped in + // silence, so the spelling is reported instead. + name: "a key only the file decode can match", + overlay: "BOT_USER = \"upper\"\n", + wantUnknown: "BOT_USER", + }, } - doc, err := New(nil).Resolve(basePath, overlayPath) - if err != nil { - t.Fatalf("Resolve: %v (an unknown key must not fail the load)", err) - } - if !slices.Contains(doc.UnknownKeys, "containers.max_concurrancy") { - t.Errorf("UnknownKeys = %v, want it to contain %q", doc.UnknownKeys, "containers.max_concurrancy") + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + basePath := filepath.Join(dir, "config.toml") + overlayPath := filepath.Join(dir, "dev.toml") + if err := os.WriteFile(basePath, []byte(minimalValidConfigTOML), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(overlayPath, []byte(tt.overlay), 0o600); err != nil { + t.Fatal(err) + } + + doc, err := New(nil).Resolve(basePath, overlayPath) + if err != nil { + t.Fatalf("Resolve: %v (an unknown key must not fail the load)", err) + } + if !slices.Contains(doc.UnknownKeys, tt.wantUnknown) { + t.Errorf("UnknownKeys = %v, want it to contain %q", doc.UnknownKeys, tt.wantUnknown) + } + }) } } diff --git a/internal/infrastructure/configuration/validate.go b/internal/infrastructure/configuration/validate.go index 61d6eef61..557c3a808 100644 --- a/internal/infrastructure/configuration/validate.go +++ b/internal/infrastructure/configuration/validate.go @@ -86,10 +86,13 @@ func validate(cfg *config.Config) error { // TOML value in one of them must not be able to fail a process's startup // (docs/prds/runtime-control-plane.md, "Bootstrap, migration, and recovery": // after migration, settings in TOML are ignored and cannot block State Store -// startup). archie-state-store resolves its file config and opens the store -// without ever layering a database resource over it (state_store.go), so a -// check left here is a check that can still block it. Those checks live in -// validateDatabaseOwned and run on the effective document instead. +// startup). archie-state-store resolves its file config and seeds the +// control-plane resources from that same document on a fresh database, and a +// seed it cannot validate is skipped rather than fatal +// (controlplane.Server.ImportConfig), so a check left here is a check that can +// still block it. Those checks live in validateDatabaseOwned and run on the +// effective document instead -- which is where the process that uses the value +// refuses it, and where the operator's fix is the file again. func validateBootstrap(cfg *config.Config) error { if err := validateForgeIntake(cfg); err != nil { return err @@ -227,7 +230,7 @@ func validatePollInterval(cfg *config.Config) error { // labels) was configured while the actual trigger-match label was left // blank. func validateDispatch(cfg *config.Config) error { - if !oneOf(cfg.Dispatch.Trigger, dispatchTriggers) { + if !DispatchTriggerValid(cfg.Dispatch.Trigger) { return fmt.Errorf("%w: dispatch.trigger %q (want %s)", ErrInvalidInput, cfg.Dispatch.Trigger, list(dispatchTriggers)) } if (cfg.Dispatch.Trigger == dispatchTriggerLabel || cfg.Dispatch.Trigger == dispatchTriggerEither) && cfg.Label == "" { @@ -236,6 +239,16 @@ func validateDispatch(cfg *config.Config) error { return nil } +// DispatchTriggerValid reports whether trigger names a discovery rule the daemon +// can poll with. It is exported because the scheduling-policy resource validator +// (internal/app/controlplane) judges the same field: that resource is seeded +// from cfg.Dispatch and replaces it wholesale once the database owns it, so a +// value this package rejects must not be storable, and a value it accepts must +// not be rejected there. One definition, both layers. +func DispatchTriggerValid(trigger string) bool { + return oneOf(trigger, dispatchTriggers) +} + // validateForgeIntake checks the forge intake mode and that webhook intake has // the secret and listen address it needs. An unset intake resolves to "poll" // without mutating cfg, so Validate (which does not apply defaults) accepts a @@ -332,9 +345,12 @@ func validateSingleIdentity(cfg *config.Config) error { } // validateRepositoryContents judges the repositories themselves, whichever list -// they appear in. Repository policies are a control-plane resource seeded from -// these, so the contents are verified on the effective document -- the one the -// database's list has already replaced the file's copy in. +// they appear in. The shared list is the decision the database's copy replaces, +// so it is verified on the effective document -- after that replacement, which is +// the only point the value being judged is the one a process will use. The +// per-identity lists are verified there for a different reason: no resource +// carries them, and the daemon is the only process that reads them +// (docs/architecture/configuration.md's IdentityConfig note). func validateRepositoryContents(cfg *config.Config) error { if len(cfg.Identities) == 0 { return validateRepos(cfg.Repos) @@ -352,15 +368,28 @@ func validateRepos(repos []config.Repo) error { if r.Owner == "" || r.Name == "" { return fmt.Errorf("%w: repos[%d] needs owner and name", ErrInvalidInput, i) } - if glob := r.ResolvedTestGlob(); glob != "" { - if _, err := filepath.Match(glob, ""); err != nil { - return fmt.Errorf("%w: repos[%d] test_glob %q: %w", ErrInvalidInput, i, glob, err) - } + if err := ValidateTestGlob(r.ResolvedTestGlob()); err != nil { + return fmt.Errorf("%w: repos[%d] %w", ErrInvalidInput, i, err) } } return nil } +// ValidateTestGlob reports whether glob is a pattern the test-protection gate can +// compile. It is exported because the repository-policies resource validator +// (internal/app/controlplane) judges the same field: that resource is seeded from +// the file's repositories and replaces them wholesale, and a pattern the gate +// cannot compile is a value that must not be storable either. +func ValidateTestGlob(glob string) error { + if glob == "" { + return nil + } + if _, err := filepath.Match(glob, ""); err != nil { + return fmt.Errorf("test_glob %q: %w", glob, err) + } + return nil +} + // effectiveNATSMode resolves an unset nats.mode from url without mutating cfg, // so validation that reads the mode never has to know whether the operator // spelled it or left it to default. validateNATS and validateContainers both From 67ebeb8d521c5ede923699963f33d1ef87f08c81 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Mon, 21 Sep 2026 20:56:00 +1000 Subject: [PATCH 4/4] fix(controlplane): a kind with no stored value leaves the file's value in force The fix-closure review found the previous pass moved the failure instead of ending it. A seed the resource validator refuses leaves the kind ABSENT, and RuntimeConfig treated an absent kind as fatal, so archied and the Gateway refused to start with "control-plane resource not found" -- a database-named error that editing config.toml cannot clear. Reproduced before the fix: every kind in the new table failed that way. Two changes, one on each side of the seam: - Client.RuntimeConfig (and RuntimeChatConfig) now treat a kind with no stored value as "the file document's value stays in effect", which is what the skip-and-report comment already claimed. The version map records no version for such a kind, so nothing is reported as applied. A value the file layer also refuses still fails closed, at configuration.Validate, with a message naming the file's own field -- and the file is the fix. - Which repository lists are valid can no longer differ between the layers. configuration.ValidateRepositories is now the single rule set (owner and name present, no duplicates, compilable test glob) and the repository-policies resource validator delegates to it. The file layer had no duplicate rule while the resource layer did, so a duplicate [[repos]] entry passed the loader and then vanished into a skipped seed. Also, from the same review: the scheduling validator's "0s" carve-out is gone, so a zero poll interval is refused at write time as configuration.Validate refuses it (there is no defaulting behind a stored value); applyOverlayFile's doc now says which overlay forms fold (a conf.d extra does not, and nothing reads cfg.Extra); and plugin-settings' no-op validator is a named function with the reason it accepts everything. --- .../app/archied/control_plane_reload_test.go | 21 ++- .../archied/control_plane_validation_test.go | 127 ++++++++++++++++-- internal/app/archied/state_store.go | 35 +++-- .../app/controlplane/operational_settings.go | 51 ++++--- internal/app/controlplane/registry.go | 6 +- internal/app/controlplane/runtime_config.go | 76 +++++++---- .../app/controlplane/validators_agree_test.go | 23 ++++ .../infrastructure/configuration/apply.go | 13 +- .../infrastructure/configuration/validate.go | 31 +++-- 9 files changed, 288 insertions(+), 95 deletions(-) diff --git a/internal/app/archied/control_plane_reload_test.go b/internal/app/archied/control_plane_reload_test.go index d2ecee6c3..6e3de5d12 100644 --- a/internal/app/archied/control_plane_reload_test.go +++ b/internal/app/archied/control_plane_reload_test.go @@ -8,6 +8,8 @@ import ( "time" "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "github.com/samcharles93/archie-core/internal/app/controlplane" "github.com/samcharles93/archie-core/internal/config" @@ -18,7 +20,9 @@ import ( // controlPlaneStub answers Query from a fixed resource map. queryErr, when // set, fails every Query so a test can drive the unreachable-State-Store -// path. +// path. A kind the map does not carry is answered the way the server answers +// one with no stored value -- codes.NotFound (controlplane.mapError) -- so a +// test can drive the absent-resource state a skipped seed leaves behind. type controlPlaneStub struct { values map[string]any queryErr error @@ -32,8 +36,12 @@ func (c *controlPlaneStub) Query(_ context.Context, request *pb.QueryRequest, _ if c.queryErr != nil { return nil, c.queryErr } - value, err := json.Marshal(c.values[request.Kind]) - return &pb.QueryResponse{Resource: &pb.Resource{Kind: request.Kind, Version: 3, ValueJson: value}}, err + value, ok := c.values[request.Kind] + if !ok { + return nil, status.Error(codes.NotFound, "resource not found") + } + encoded, err := json.Marshal(value) + return &pb.QueryResponse{Resource: &pb.Resource{Kind: request.Kind, Version: 3, ValueJson: encoded}}, err } func (*controlPlaneStub) History(context.Context, *pb.HistoryRequest, ...grpc.CallOption) (*pb.HistoryResponse, error) { @@ -67,11 +75,15 @@ func databaseOwnedResources() map[string]any { // fileConfig is what the loader produces from config.toml alone: the layer a // SIGHUP reload re-resolves, holding none of the database's values. It is a // config the daemon would run, so it passes configuration.Validate -- the live -// apply path runs that same check over the snapshot it is about to publish. +// apply path runs that same check over the snapshot it is about to publish -- +// and every field validation reads is filled in the way the loader's defaults +// fill it, so a kind with no stored value can fall back to this document (see +// TestRuntimeConfigKeepsTheFileValueWhenAKindHasNoStoredResource). func fileConfig() config.Config { return config.Config{ BotUser: "widget", Forge: config.Forge{Type: "github"}, + Dispatch: config.Dispatch{Trigger: "assignee"}, Providers: map[string]config.Provider{"file": {Class: "openai"}}, Models: map[string]string{"builder": "file/model"}, Repos: []config.Repo{{Owner: "acme", Name: "from-file"}}, @@ -81,7 +93,6 @@ func fileConfig() config.Config { PluginDir: "/file/plugins", Containers: config.ContainerConfig{Image: "archie:from-file", PullPolicy: "missing"}, Budgets: config.Budgets{MaxSteps: 1, WallClock: config.Duration(time.Minute)}, - Dispatch: config.Dispatch{Trigger: "assignee"}, } } diff --git a/internal/app/archied/control_plane_validation_test.go b/internal/app/archied/control_plane_validation_test.go index 856a172e5..1ab96864a 100644 --- a/internal/app/archied/control_plane_validation_test.go +++ b/internal/app/archied/control_plane_validation_test.go @@ -7,10 +7,12 @@ import ( "log/slog" "os" "path/filepath" + "strings" "testing" "time" "github.com/samcharles93/archie-core/internal/app/controlplane" + "github.com/samcharles93/archie-core/internal/config" "github.com/samcharles93/archie-core/internal/infrastructure/configuration" ) @@ -66,21 +68,29 @@ func TestLoadConfigAcceptsStaleDatabaseOwnedValues(t *testing.T) { } } +// staleBootGrace is how long one boot in +// TestStateStoreBootsWithStaleDatabaseOwnedValues may run before the test cancels +// it: short, because the assertion does not depend on it -- a refusal arrives as +// an error either before the cancel (first select) or after it (second select, +// where a non-Canceled error fails the test). +// +// staleShutdownGrace bounds the shutdown, and is deliberately not the 5s the +// single-boot regression test uses: this table runs five boots back to back on a +// host that may be busy, and the select returns the moment the process does, so +// the bound costs nothing when shutdown is quick and only decides how long a +// failure takes to report. +const ( + staleBootGrace = 250 * time.Millisecond + staleShutdownGrace = 30 * time.Second +) + // TestStateStoreBootsWithStaleDatabaseOwnedValues is the process-level half of // the item, and the layer that actually decides it: archie-state-store seeds // every control-plane resource from the same file config on a fresh database // (state_store.go, control.ImportConfig), and that seed is what runs the // resource's own validator. A seed that does not validate is skipped and -// reported, not fatal -- the value stays file-owned, archied refuses to run with -// it, and the operator's fix in config.toml is what clears it. -// staleBootGrace is how long one boot in -// TestStateStoreBootsWithStaleDatabaseOwnedValues may run before the test -// cancels it. It can be short because the assertion does not depend on it: a -// refusal arrives as an error either before the cancel (first select) or after -// it (second select, where a non-Canceled error fails the test). The wait only -// gives the serving path a chance to prove it did not return immediately. -const staleBootGrace = 250 * time.Millisecond - +// reported, not fatal -- the kind stays absent, the file's value is the one in +// effect, and the operator's fix in config.toml is what clears it. func TestStateStoreBootsWithStaleDatabaseOwnedValues(t *testing.T) { for _, tt := range staleDatabaseOwnedSettings { t.Run(tt.name, func(t *testing.T) { @@ -116,13 +126,108 @@ func TestStateStoreBootsWithStaleDatabaseOwnedValues(t *testing.T) { if err != nil && !errors.Is(err, context.Canceled) { t.Fatalf("RunStateStore returned %v, want nil or context.Canceled", err) } - case <-time.After(shutdownGrace): + case <-time.After(staleShutdownGrace): t.Fatal("RunStateStore did not return within the shutdown grace after cancellation") } }) } } +// TestRuntimeConfigKeepsTheFileValueWhenAKindHasNoStoredResource is the other +// end of a seed the control plane refused: the kind stays ABSENT rather than +// being stored (ImportConfig skips it), and boot must then fall back to the +// file document's value. Treating an absent kind as fatal is how a config the +// file layer blesses left the daemon unbootable with a database-named error, +// and left editing config.toml unable to fix it. +func TestRuntimeConfigKeepsTheFileValueWhenAKindHasNoStoredResource(t *testing.T) { + tests := []struct { + name string + kind string + check func(t *testing.T, cfg config.Config) + }{ + { + name: "repository policies", + kind: controlplane.RepositoryPoliciesKind, + check: func(t *testing.T, cfg config.Config) { + t.Helper() + if len(cfg.Repos) != 1 || cfg.Repos[0].Name != "from-file" { + t.Errorf("Repos = %+v, want the file's from-file", cfg.Repos) + } + }, + }, + { + name: "scheduling policy", + kind: controlplane.SchedulingPolicyKind, + check: func(t *testing.T, cfg config.Config) { + t.Helper() + if cfg.PollInterval != config.Duration(time.Minute) { + t.Errorf("PollInterval = %v, want the file's 1m", cfg.PollInterval) + } + }, + }, + { + name: "container runtime policies", + kind: controlplane.ContainerRuntimePoliciesKind, + check: func(t *testing.T, cfg config.Config) { + t.Helper() + if cfg.Containers.Image != "archie:from-file" { + t.Errorf("Containers.Image = %q, want the file's archie:from-file", cfg.Containers.Image) + } + }, + }, + { + name: "providers", + kind: controlplane.ProviderSettingsKind, + check: func(t *testing.T, cfg config.Config) { + t.Helper() + if got := cfg.Providers["file"].Class; got != "openai" { + t.Errorf("Providers[file].Class = %q, want the file's openai", got) + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + resources := databaseOwnedResources() + delete(resources, tt.kind) + b := newReloadBoot(t, &controlPlaneStub{values: resources}) + + if err := b.loadRuntimeConfig(t.Context()); err != nil { + t.Fatalf("loadRuntimeConfig: %v (a kind with no stored value must leave the file's value in effect)", err) + } + tt.check(t, b.cfgHolder.Get()) + }) + } +} + +// TestDuplicateRepositoriesFailOnTheFilesOwnMessage is the end of the path the +// fix-closure review found. A duplicate [[repos]] entry is refused by the +// repository-policies validator, so the kind is never seeded and stays absent -- +// and boot must then fail on the FILE's copy of the value, naming the duplicate, +// with the file as the fix. It used to fail on "control-plane resource not +// found": a database-named error that editing config.toml could not clear. +func TestDuplicateRepositoriesFailOnTheFilesOwnMessage(t *testing.T) { + resources := databaseOwnedResources() + delete(resources, controlplane.RepositoryPoliciesKind) + b := newReloadBoot(t, &controlPlaneStub{values: resources}) + b.cfg.Repos = []config.Repo{{Owner: "acme", Name: "app"}, {Owner: "acme", Name: "app"}} + + err := b.loadRuntimeConfig(t.Context()) + if err == nil { + t.Fatal("loadRuntimeConfig accepted a duplicate repository list") + } + if !errors.Is(err, configuration.ErrInvalidInput) { + t.Errorf("loadRuntimeConfig = %v, want it to wrap configuration.ErrInvalidInput", err) + } + if !strings.Contains(err.Error(), "duplicates") { + t.Errorf("loadRuntimeConfig = %v, want the duplicate named the way configuration.Validate names it", err) + } + if strings.Contains(err.Error(), "not found") { + t.Errorf("loadRuntimeConfig = %v, want no absent-resource error: the file's value is the one in effect", err) + } +} + // TestRuntimeConfigFailsClosedOnAnInvalidDatabaseValue is the other half of the // same change: the checks that no longer run at load run on the effective // document, at the one place database resources are layered in. A stored value diff --git a/internal/app/archied/state_store.go b/internal/app/archied/state_store.go index 3ba9e9cb5..474873b59 100644 --- a/internal/app/archied/state_store.go +++ b/internal/app/archied/state_store.go @@ -110,17 +110,7 @@ func RunStateStore(ctx context.Context, options StateStoreOptions) error { if err != nil { return fmt.Errorf("import control-plane resources: %w", err) } - for _, skip := range skipped { - // Reported, not fatal: a stale TOML value in a setting the database owns - // must not stop this process from starting - // (docs/prds/runtime-control-plane.md, "Bootstrap, migration, and - // recovery"), and the resource stays absent, so the file's value is - // still the one in effect. Whether it may RUN is decided where it is - // used: boot.runtimeConfig validates the effective document and refuses - // to start archied or the Gateway with it. - b.log.Error("control-plane resource not seeded; the file config's value stays in effect", - "kind", skip.Kind, "err", skip.Err) - } + b.reportUnseededResources(skipped) if err := migrateLegacySchedules(ctx, control, b.cfg.DBPath, versions[controlplane.SchedulesKind]); err != nil { return fmt.Errorf("migrate schedules: %w", err) } @@ -153,6 +143,29 @@ func RunStateStore(ctx context.Context, options StateStoreOptions) error { return serveStateStore(ctx, listener, deps, opts) } +// reportUnseededResources logs every kind ImportConfig could not seed. +// +// Reported, not fatal: a stale TOML value in a setting the database owns must +// not stop this process from starting (docs/prds/runtime-control-plane.md, +// "Bootstrap, migration, and recovery"). The kind stays ABSENT rather than +// half-written, so the file document's value is the one in effect for every +// reader -- the layering leaves a kind with no stored value alone +// (controlplane.resourceReader) -- and the operator's fix is still the file. +// Whether the value may RUN is decided where it is used: boot.runtimeConfig +// validates the effective document and refuses to start archied or the Gateway +// with one it cannot run. The seed is retried on this process's next start, so +// a corrected config.toml takes effect when archie-state-store restarts. +// +// It is a method rather than a loop in RunStateStore because the boot sequence +// is at its complexity budget: the branch belongs to the report's own policy, +// not to the sequence that triggers it. +func (b *boot) reportUnseededResources(skipped []controlplane.SeedSkip) { + for _, skip := range skipped { + b.log.Error("control-plane resource not seeded; the kind stays absent and the file's value is the one in effect", + "kind", skip.Kind, "err", skip.Err) + } +} + // openStateStoreControlPlane builds the State Store's control plane server: the // validating side of the workflow step vocabulary, registered here at the // composition root before the first definition is read or replaced. The same diff --git a/internal/app/controlplane/operational_settings.go b/internal/app/controlplane/operational_settings.go index 03b9d4591..e439943f8 100644 --- a/internal/app/controlplane/operational_settings.go +++ b/internal/app/controlplane/operational_settings.go @@ -41,33 +41,27 @@ func operationalDefinitions() []Definition { {Kind: ToolSettingsKind, Title: "Tool and MCP settings", ApplyMode: "restart-required", Schema: objectSchema, Seed: seedTools, Validate: validateTools}, {Kind: PluginSettingsKind, Title: "Plugin settings", ApplyMode: "restart-required", Schema: objectSchema, Seed: func(cfg config.Config) any { return pluginSettings{cfg.PluginDir, cfg.ModuleDir, cfg.SecretEngineDir, cfg.SkillsDir} - }, Validate: func(input []byte) error { return validateAs(input, func(pluginSettings) error { return nil }) }}, + }, Validate: validatePluginSettings}, {Kind: ContainerRuntimePoliciesKind, Title: "Container runtime policies", ApplyMode: "restart-required", Schema: objectSchema, Seed: func(cfg config.Config) any { return cfg.Containers }, Validate: validateContainers}, } } +// validatePluginSettings accepts every value: the four directories are free-form +// operator paths with no cross-field rule and no rule the configuration package +// applies either (configuration.Validate has no say over them), so there is +// nothing yet for this validator to enforce. It stays a real function rather than +// an inline no-op so the place to add a rule is obvious when one exists -- e.g. +// "a path that is absolute" or "a skills dir under the plugin dir". +func validatePluginSettings(input []byte) error { + return validateAs(input, func(pluginSettings) error { return nil }) +} + func validateRepositories(input []byte) error { - return validateAs(input, func(repos []config.Repo) error { - seen := make(map[string]struct{}, len(repos)) - for _, repo := range repos { - if strings.TrimSpace(repo.Owner) == "" || strings.TrimSpace(repo.Name) == "" { - return fmt.Errorf("repository owner and name are required") - } - if _, exists := seen[repo.FullName()]; exists { - return fmt.Errorf("duplicate repository %q", repo.FullName()) - } - seen[repo.FullName()] = struct{}{} - // The repository-policies resource replaces the file's list, so the - // rules that judge the file's list have to hold here too. They come - // from the configuration package rather than being restated: a - // resource that accepts what configuration.Validate rejects becomes a - // stored value the operator cannot clear by editing config.toml. - if err := configuration.ValidateTestGlob(repo.ResolvedTestGlob()); err != nil { - return err - } - } - return nil - }) + // The rules live in the configuration package and are shared with the file + // document's own validation (configuration.ValidateRepositories): which + // repository lists are valid cannot differ between the file that seeds this + // resource and the resource that replaces it. + return validateAs(input, configuration.ValidateRepositories) } func validateScheduling(input []byte) error { @@ -75,13 +69,14 @@ func validateScheduling(input []byte) error { if policy.MaxRetries < 0 { return fmt.Errorf("max_retries must not be negative") } - if policy.PollInterval != "0s" { - interval, err := time.ParseDuration(policy.PollInterval) - if err != nil || interval < 0 { - return fmt.Errorf("poll_interval must be a non-negative duration") - } + // A positive interval, not a non-negative one: the file layer's rule is + // the same, and there is no defaulting behind a stored value -- a stored + // "0s" reaches cfg.PollInterval as zero and stops archied starting. + interval, err := time.ParseDuration(policy.PollInterval) + if err != nil || interval <= 0 { + return fmt.Errorf("poll_interval must be a positive duration") } - // Same reason as the repository glob above: RuntimeConfig replaces + // Same reason as the repository rules above: RuntimeConfig replaces // cfg.Dispatch from this resource, and the daemon refuses to start with a // trigger it does not know. if !configuration.DispatchTriggerValid(policy.Dispatch.Trigger) { diff --git a/internal/app/controlplane/registry.go b/internal/app/controlplane/registry.go index b7ecf168a..86648d0e6 100644 --- a/internal/app/controlplane/registry.go +++ b/internal/app/controlplane/registry.go @@ -72,8 +72,10 @@ type SeedSkip struct { // from the file config, and reports the version of every kind it left alone. // // A seed the resource's own validator refuses is SKIPPED and returned in skipped, -// not fatal. The setting it would have carried stays file-owned, so the value in -// effect is still the file's, and the fix is still the file's. +// not fatal. Nothing is written for that kind, so it stays ABSENT and the file +// document's value stays the one in effect (Client.RuntimeConfig leaves an absent +// kind alone), with the fix still in the file. The seed is retried on the next +// start of this process, so correcting config.toml re-seeds it. // docs/prds/runtime-control-plane.md, "Bootstrap, migration, and recovery": // after migration, settings in TOML are ignored and cannot block State Store // startup -- and this import runs on the State Store's startup path. Fail closed diff --git a/internal/app/controlplane/runtime_config.go b/internal/app/controlplane/runtime_config.go index 0bd90c6c5..e09694cf7 100644 --- a/internal/app/controlplane/runtime_config.go +++ b/internal/app/controlplane/runtime_config.go @@ -13,12 +13,21 @@ import ( ) // resourceReader is the one read the layering performs: a resource kind's value -// and the version it came from. The gRPC client and the in-process server both -// provide it, so "what the database covers in the running config" has exactly -// one implementation. The offline validate runs the same code as boot, which is -// the only way its verdict can be boot's verdict. +// and the version it came from, or nothing at all when the store holds no value +// for that kind. The gRPC client and the in-process server both provide it, so +// "what the database covers in the running config" has exactly one +// implementation. The offline validate runs the same code as boot, which is the +// only way its verdict can be boot's verdict. +// +// found is false only for the reader that dials a live State Store: a kind with +// no stored value leaves the file document's value in effect instead of failing +// the process, which is the state a seed ImportConfig refused leaves behind +// (controlplane.Server.ImportConfig skips it) and the state of a database the +// migration has not reached yet. storeReader never reports absent -- it answers +// with the seed the State Store would write -- so the offline verdict stays +// boot's. type resourceReader interface { - query(ctx context.Context, kind string, decode func([]byte) error) (int64, error) + query(ctx context.Context, kind string, decode func([]byte) error) (int64, bool, error) } // RuntimeConfig applies restart-scoped database resources over bootstrap @@ -67,27 +76,28 @@ type storeReader struct { seeds map[string][]byte } -func (r storeReader) query(ctx context.Context, kind string, decode func([]byte) error) (int64, error) { +func (r storeReader) query(ctx context.Context, kind string, decode func([]byte) error) (int64, bool, error) { resource, err := r.resources.Resource(ctx, kind) if errors.Is(err, store.ErrResourceNotFound) { seed, ok := r.seeds[kind] if !ok { - return 0, fmt.Errorf("read %s: %w", kind, store.ErrResourceNotFound) + return 0, false, fmt.Errorf("read %s: %w", kind, store.ErrResourceNotFound) } if err := decode(seed); err != nil { - return 0, fmt.Errorf("decode %s seed: %w", kind, err) + return 0, false, fmt.Errorf("decode %s seed: %w", kind, err) } // No version: nothing has been stored for this kind, so nothing has - // been applied either. - return 0, nil + // been applied either. It is still a value to layer in -- the seed -- + // which is why this reader never reports the kind absent. + return 0, true, nil } if err != nil { - return 0, fmt.Errorf("read %s: %w", kind, err) + return 0, false, fmt.Errorf("read %s: %w", kind, err) } if err := decode(resource.Value); err != nil { - return 0, fmt.Errorf("decode %s: %w", kind, err) + return 0, false, fmt.Errorf("decode %s: %w", kind, err) } - return resource.Version, nil + return resource.Version, true, nil } func runtimeConfigFrom(ctx context.Context, reader resourceReader, base config.Config) (config.Config, map[string]int64, error) { @@ -171,7 +181,7 @@ func runtimeToolConfigFrom(ctx context.Context, reader resourceReader, versions func runtimeChatConfigFrom(ctx context.Context, reader resourceReader, base config.ChatConfig) (config.ChatConfig, int64, error) { out := base - version, err := reader.query(ctx, ChannelSettingsKind, func(value []byte) error { + version, found, err := reader.query(ctx, ChannelSettingsKind, func(value []byte) error { var settings channelSettings if err := json.Unmarshal(value, &settings); err != nil { return err @@ -186,17 +196,31 @@ func runtimeChatConfigFrom(ctx context.Context, reader resourceReader, base conf } return nil }) - return out, version, err + if err != nil { + return out, 0, err + } + if !found { + return out, 0, nil + } + return out, version, nil } // layerResource decodes a resource and records the version it came from, so the // layering ends up holding the version of every kind it applied. +// +// A kind with no stored value is not an error and records no version: the file +// document's value stays in effect. That is the state a seed the resource +// validator refused leaves behind (controlplane.Server.ImportConfig skips it), +// and failing here instead would stop the process with a database-named error +// that editing config.toml cannot clear. func layerResource(ctx context.Context, reader resourceReader, versions map[string]int64, kind string, decode func([]byte) error) error { - version, err := reader.query(ctx, kind, decode) + version, found, err := reader.query(ctx, kind, decode) if err != nil { return err } - versions[kind] = version + if found { + versions[kind] = version + } return nil } @@ -205,17 +229,23 @@ func layerResourceJSON(ctx context.Context, reader resourceReader, versions map[ } // query returns the version of the resource it decoded, so callers that layer -// a resource in can report the version they applied. -func (c *Client) query(ctx context.Context, kind string, decode func([]byte) error) (int64, error) { +// a resource in can report the version they applied. A kind the store holds no +// value for is reported as not found rather than as an error: the caller leaves +// the file document's value in effect (see resourceReader). +func (c *Client) query(ctx context.Context, kind string, decode func([]byte) error) (int64, bool, error) { response, err := c.rpc.Query(ctx, &pb.QueryRequest{Kind: kind}) if err != nil { - return 0, clientError(err) + mapped := clientError(err) + if errors.Is(mapped, ErrNotFound) { + return 0, false, nil + } + return 0, false, mapped } if response.Resource == nil { - return 0, fmt.Errorf("%s resource missing", kind) + return 0, false, fmt.Errorf("%s resource missing", kind) } if err := decode(response.Resource.ValueJson); err != nil { - return 0, fmt.Errorf("decode %s: %w", kind, err) + return 0, false, fmt.Errorf("decode %s: %w", kind, err) } - return response.Resource.Version, nil + return response.Resource.Version, true, nil } diff --git a/internal/app/controlplane/validators_agree_test.go b/internal/app/controlplane/validators_agree_test.go index 814a058ed..f3e6f890d 100644 --- a/internal/app/controlplane/validators_agree_test.go +++ b/internal/app/controlplane/validators_agree_test.go @@ -48,6 +48,17 @@ func TestResourceValidatorsRejectWhatEffectiveValidationRejects(t *testing.T) { mutate: func(cfg *config.Config) { cfg.PollInterval = config.Duration(-5 * time.Second) }, wantErr: true, }, + { + // A zero interval is what applyDefaults replaces before validation + // runs, so the file layer only ever sees it as a value to reject. The + // resource has no defaulting behind it: a stored "0s" would become + // cfg.PollInterval = 0 and stop archied starting. + name: "zero poll interval", + kind: SchedulingPolicyKind, + value: map[string]any{"poll_interval": "0s", "max_retries": 0, "dispatch": map[string]any{"trigger": "assignee"}}, + mutate: func(cfg *config.Config) { cfg.PollInterval = 0 }, + wantErr: true, + }, { name: "valid repository policies", kind: RepositoryPoliciesKind, @@ -62,6 +73,18 @@ func TestResourceValidatorsRejectWhatEffectiveValidationRejects(t *testing.T) { mutate: func(cfg *config.Config) { cfg.Repos = []config.Repo{{Owner: "acme", Name: "app", TestGlob: "["}} }, wantErr: true, }, + { + // Both layers have to call a duplicate list invalid. When only the + // resource did, a config the file layer blessed left the daemon + // unbootable on a database-named error. + name: "duplicate repository", + kind: RepositoryPoliciesKind, + value: []config.Repo{{Owner: "acme", Name: "app"}, {Owner: "acme", Name: "app"}}, + mutate: func(cfg *config.Config) { + cfg.Repos = []config.Repo{{Owner: "acme", Name: "app"}, {Owner: "acme", Name: "app"}} + }, + wantErr: true, + }, { name: "valid container runtime policies", kind: ContainerRuntimePoliciesKind, diff --git a/internal/infrastructure/configuration/apply.go b/internal/infrastructure/configuration/apply.go index d7d900151..73da4b275 100644 --- a/internal/infrastructure/configuration/apply.go +++ b/internal/infrastructure/configuration/apply.go @@ -35,11 +35,14 @@ func ApplyOverlayValues(cfg *config.Config, overrides map[string]any) error { // file's keys that nothing consumes. target is whatever the overlay addresses: // the whole config for a main file, one sub-struct for a feature file. // -// Both overlay forms go through it -- the single overlay file, and every file an -// overlay directory contributes -- so neither form can lose the fields of a -// map-valued entry it only partly addresses. Decoding an overlay into target -// directly replaces such an entry wholesale, which is how an overlay naming only -// services.state.target cleared the target_token the base file set. +// Both processed overlay forms go through it -- the single overlay file, and an +// overlay directory's main and feature files -- so neither can lose the fields of +// a map-valued entry it only partly addresses. A conf.d/*.yaml extra in an +// overlay directory does not: decodeExtra stores a whole file under +// cfg.Extra[name] and nothing reads that map, so there is no entry to fold into. +// Decoding an overlay into target directly replaces such an entry wholesale, +// which is how an overlay naming only services.state.target cleared the +// target_token the base file set. func applyOverlayFile(path string, target any) ([]string, error) { mapping, err := decodeFileMapping(path) if err != nil { diff --git a/internal/infrastructure/configuration/validate.go b/internal/infrastructure/configuration/validate.go index 557c3a808..6145c1ae9 100644 --- a/internal/infrastructure/configuration/validate.go +++ b/internal/infrastructure/configuration/validate.go @@ -353,34 +353,45 @@ func validateSingleIdentity(cfg *config.Config) error { // (docs/architecture/configuration.md's IdentityConfig note). func validateRepositoryContents(cfg *config.Config) error { if len(cfg.Identities) == 0 { - return validateRepos(cfg.Repos) + return ValidateRepositories(cfg.Repos) } for i, id := range cfg.Identities { - if err := validateRepos(id.Repos); err != nil { + if err := ValidateRepositories(id.Repos); err != nil { return fmt.Errorf("identities[%d]: %w", i, err) } } return nil } -func validateRepos(repos []config.Repo) error { +// ValidateRepositories judges a repository list: owner and name present, no +// duplicate entries, and a test glob the gate can compile. It is exported +// because the repository-policies resource validator (internal/app/controlplane) +// judges the same list -- that resource is seeded from these and replaces them +// wholesale, so the two layers must not disagree about which lists are valid, in +// either direction: a list one layer accepts and the other refuses is a config +// that boots file-side and stores nowhere, or a stored value the daemon then +// refuses to start with. +func ValidateRepositories(repos []config.Repo) error { + seen := make(map[string]struct{}, len(repos)) for i, r := range repos { if r.Owner == "" || r.Name == "" { return fmt.Errorf("%w: repos[%d] needs owner and name", ErrInvalidInput, i) } - if err := ValidateTestGlob(r.ResolvedTestGlob()); err != nil { + if _, exists := seen[r.FullName()]; exists { + return fmt.Errorf("%w: repos[%d] duplicates %q", ErrInvalidInput, i, r.FullName()) + } + seen[r.FullName()] = struct{}{} + if err := validateTestGlob(r.ResolvedTestGlob()); err != nil { return fmt.Errorf("%w: repos[%d] %w", ErrInvalidInput, i, err) } } return nil } -// ValidateTestGlob reports whether glob is a pattern the test-protection gate can -// compile. It is exported because the repository-policies resource validator -// (internal/app/controlplane) judges the same field: that resource is seeded from -// the file's repositories and replaces them wholesale, and a pattern the gate -// cannot compile is a value that must not be storable either. -func ValidateTestGlob(glob string) error { +// validateTestGlob reports whether glob is a pattern the test-protection gate can +// compile. It has one caller on each side through ValidateRepositories, so the +// rule has one definition rather than a copy per layer. +func validateTestGlob(glob string) error { if glob == "" { return nil }