From 9a45a35c27de4e9c26278d36e332dfb7afc2d889 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Mon, 21 Sep 2026 20:39:45 +1000 Subject: [PATCH] fix(config): judge a database-owned setting at the layer that owns it Review of the two previous commits found the split was incomplete in both directions: the checks moved off the file load path were still enforced anywhere the value could enter the system, and the fold reached only one of the three overlay paths. A stale database-owned TOML value could still stop archie-state-store. The process seeds every control-plane resource from the same file config on a fresh database, and the seed runs the resource's own validator; three of the five values the loader now accepts aborted the boot there (verified by booting RunStateStore before the fix). A seed that does not validate is now SKIPPED and reported instead of fatal: the resource stays absent, so the file's value is the one in effect and editing config.toml is still the fix -- the daemon and the Gateway are where it fails closed, at boot.runtimeConfig. The resource validators were weaker than configuration.Validate for exactly the fields the split reclassified, so a value the file path used to refuse at load became storable in the database, and then un-fixable by editing config.toml. The trigger enum and the test-glob rule now come from the configuration package rather than being restated, the container image is required there as it is in the file, and a table test ties the two layers together value by value. The directory form of -config-overlay kept the archie-core-e2e2 failure: it decoded each overlay file straight into the populated config, so a map-valued entry it only partly addressed still cleared the fields it did not name. Both the overlay layer's main file and its feature files now fold, which is what Loader.Dir's documented parity with Loader.Overlay has always claimed. The overlay file's key report is derived from the apply's own view of the file as well as the file decode's, because TOML matches keys case-insensitively and the fold's yaml decode does not: a key only the file decode matched was consumed by the report and dropped by the apply, a silent no-op that worked in the base file. Tests: the state-store boot is asserted against the real process, the directory overlay and the case-mismatched key each have one, and ImportConfig pins that a refused seed is not stored. --- cmd/archie-state-store/recovery_test.go | 10 +- .../archied/control_plane_validation_test.go | 136 ++++++++++++---- internal/app/archied/state_store.go | 13 +- .../app/controlplane/controlplane_test.go | 61 ++++++- .../app/controlplane/operational_settings.go | 20 +++ internal/app/controlplane/registry.go | 38 ++++- .../app/controlplane/validators_agree_test.go | 150 ++++++++++++++++++ .../infrastructure/configuration/apply.go | 60 +++++-- .../infrastructure/configuration/decode.go | 22 ++- .../infrastructure/configuration/loader.go | 98 ++++++++++-- .../configuration/loader_test.go | 127 ++++++++++++--- .../configuration/unknown_keys_test.go | 65 +++++--- .../infrastructure/configuration/validate.go | 53 +++++-- 13 files changed, 710 insertions(+), 143 deletions(-) create mode 100644 internal/app/controlplane/validators_agree_test.go diff --git a/cmd/archie-state-store/recovery_test.go b/cmd/archie-state-store/recovery_test.go index b58629f29..0f0359b31 100644 --- a/cmd/archie-state-store/recovery_test.go +++ b/cmd/archie-state-store/recovery_test.go @@ -708,9 +708,17 @@ func seedStoreResources(t *testing.T, st *store.Store, configPath string) { if err != nil { t.Fatalf("build control plane server: %v", err) } - if _, err := server.ImportConfig(t.Context(), doc.Config); err != nil { + _, skipped, err := server.ImportConfig(t.Context(), doc.Config) + if err != nil { t.Fatalf("seed store resources: %v", err) } + if len(skipped) > 0 { + // This helper stands in for the State Store's own seeding, which seeds + // every kind it can and leaves the rest absent. A skipped kind here + // would mean the fixture store is missing a resource it claims to hold, + // and the verdict under test would be about a different store. + t.Fatalf("seed store resources: %d kind(s) refused: %+v", len(skipped), skipped) + } } // storedResourceCount reads the count validate reports out of its summary line. diff --git a/internal/app/archied/control_plane_validation_test.go b/internal/app/archied/control_plane_validation_test.go index 1a28fbf4a..856a172e5 100644 --- a/internal/app/archied/control_plane_validation_test.go +++ b/internal/app/archied/control_plane_validation_test.go @@ -1,50 +1,59 @@ package archied import ( + "context" "errors" + "fmt" "log/slog" + "os" + "path/filepath" "testing" + "time" "github.com/samcharles93/archie-core/internal/app/controlplane" "github.com/samcharles93/archie-core/internal/infrastructure/configuration" ) -// TestLoadConfigAcceptsStaleDatabaseOwnedValues is archie-core-i3qm at the boot -// entry point. docs/prds/runtime-control-plane.md, "Bootstrap, migration, and -// recovery": after migration, settings in TOML are ignored and cannot block -// State Store startup. archie-state-store boots through loadConfig and then -// opens the store; it never layers a control-plane resource over the file -// document (state_store.go), so a stale value in a setting the control plane -// owns must not fail boot. Every value below is one a control-plane resource -// replaces wholesale. -func TestLoadConfigAcceptsStaleDatabaseOwnedValues(t *testing.T) { - tests := []struct { - name string - body string - }{ - { - name: "provider base_url carrying userinfo", - body: "[providers.openai]\nclass = \"openai\"\nbase_url = \"https://token@example.com/v1\"\n", - }, - { - name: "negative poll_interval", - body: "poll_interval = \"-5s\"\n", - }, - { - name: "unrecognised dispatch.trigger", - body: "[dispatch]\ntrigger = \"labels\"\n", - }, - { - name: "negative containers.volume_ttl", - body: "[containers]\nvolume_ttl = \"-1m\"\n", - }, - { - name: "malformed repos test_glob", - body: "[[repos]]\nowner = \"acme\"\nname = \"app\"\ntest_glob = \"[\"\n", - }, - } +// staleDatabaseOwnedSettings are TOML fragments, each carrying one value a +// control-plane resource is seeded from and replaces wholesale. They are the +// values docs/prds/runtime-control-plane.md, +// "Bootstrap, migration, and recovery" says must not block a process: "after +// migration, settings in TOML are ignored and cannot block State Store +// startup". +var staleDatabaseOwnedSettings = []struct { + name string + body string +}{ + { + name: "provider base_url carrying userinfo", + body: "[providers.openai]\nclass = \"openai\"\nbase_url = \"https://token@example.com/v1\"\n", + }, + { + name: "negative poll_interval", + body: "poll_interval = \"-5s\"\n", + }, + { + name: "unrecognised dispatch.trigger", + body: "[dispatch]\ntrigger = \"labels\"\n", + }, + { + name: "negative containers.volume_ttl", + body: "[containers]\nvolume_ttl = \"-1m\"\n", + }, + { + name: "malformed repos test_glob", + body: "[[repos]]\nowner = \"acme\"\nname = \"app\"\ntest_glob = \"[\"\n", + }, +} - for _, tt := range tests { +// TestLoadConfigAcceptsStaleDatabaseOwnedValues is the first of the two layers +// archie-state-store boots through. loadConfig resolves the file and is where +// the loader used to reject these values; it must not, because the setting +// belongs to the control plane. The second layer is the boot itself, covered by +// TestStateStoreBootsWithStaleDatabaseOwnedValues -- passing only this one is +// not the item's claim. +func TestLoadConfigAcceptsStaleDatabaseOwnedValues(t *testing.T) { + for _, tt := range staleDatabaseOwnedSettings { t.Run(tt.name, func(t *testing.T) { path := t.TempDir() + "/config.toml" writeConfig(t, path, "bot_user = \"widget\"\n"+tt.body) @@ -57,6 +66,63 @@ func TestLoadConfigAcceptsStaleDatabaseOwnedValues(t *testing.T) { } } +// TestStateStoreBootsWithStaleDatabaseOwnedValues is the process-level half of +// the item, and the layer that actually decides it: archie-state-store seeds +// every control-plane resource from the same file config on a fresh database +// (state_store.go, control.ImportConfig), and that seed is what runs the +// resource's own validator. A seed that does not validate is skipped and +// reported, not fatal -- the value stays file-owned, archied refuses to run with +// it, and the operator's fix in config.toml is what clears it. +// staleBootGrace is how long one boot in +// TestStateStoreBootsWithStaleDatabaseOwnedValues may run before the test +// cancels it. It can be short because the assertion does not depend on it: a +// refusal arrives as an error either before the cancel (first select) or after +// it (second select, where a non-Canceled error fails the test). The wait only +// gives the serving path a chance to prove it did not return immediately. +const staleBootGrace = 250 * time.Millisecond + +func TestStateStoreBootsWithStaleDatabaseOwnedValues(t *testing.T) { + for _, tt := range staleDatabaseOwnedSettings { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.toml") + body := fmt.Sprintf( + "bot_user = 'archie'\ndb_path = %q\n%s[forge]\ntype = 'github'\nhost = 'https://github.example.com'\n", + filepath.Join(dir, "state.db"), tt.body, + ) + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + + // Loopback, so no token is required. Cancellation is the assertion: + // a boot that refuses the config returns before it, and one that + // serves returns only when the context ends. Cancel first, then + // assert -- racing startup against a wall clock is a coin flip. + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + errCh := make(chan error, 1) + go func() { + errCh <- RunStateStore(ctx, StateStoreOptions{Config: path, Listen: "127.0.0.1:0"}) + }() + + select { + case err := <-errCh: + t.Fatalf("RunStateStore returned before cancellation, want it to serve: %v", err) + case <-time.After(staleBootGrace): + } + cancel() + select { + case err := <-errCh: + if err != nil && !errors.Is(err, context.Canceled) { + t.Fatalf("RunStateStore returned %v, want nil or context.Canceled", err) + } + case <-time.After(shutdownGrace): + t.Fatal("RunStateStore did not return within the shutdown grace after cancellation") + } + }) + } +} + // TestRuntimeConfigFailsClosedOnAnInvalidDatabaseValue is the other half of the // same change: the checks that no longer run at load run on the effective // document, at the one place database resources are layered in. A stored value diff --git a/internal/app/archied/state_store.go b/internal/app/archied/state_store.go index b1b919d35..3ba9e9cb5 100644 --- a/internal/app/archied/state_store.go +++ b/internal/app/archied/state_store.go @@ -106,10 +106,21 @@ func RunStateStore(ctx context.Context, options StateStoreOptions) error { if err != nil { return err } - versions, err := control.ImportConfig(ctx, b.cfg) + versions, skipped, err := control.ImportConfig(ctx, b.cfg) if err != nil { return fmt.Errorf("import control-plane resources: %w", err) } + for _, skip := range skipped { + // Reported, not fatal: a stale TOML value in a setting the database owns + // must not stop this process from starting + // (docs/prds/runtime-control-plane.md, "Bootstrap, migration, and + // recovery"), and the resource stays absent, so the file's value is + // still the one in effect. Whether it may RUN is decided where it is + // used: boot.runtimeConfig validates the effective document and refuses + // to start archied or the Gateway with it. + b.log.Error("control-plane resource not seeded; the file config's value stays in effect", + "kind", skip.Kind, "err", skip.Err) + } if err := migrateLegacySchedules(ctx, control, b.cfg.DBPath, versions[controlplane.SchedulesKind]); err != nil { return fmt.Errorf("migrate schedules: %w", err) } diff --git a/internal/app/controlplane/controlplane_test.go b/internal/app/controlplane/controlplane_test.go index 499ce2c61..fcbe17061 100644 --- a/internal/app/controlplane/controlplane_test.go +++ b/internal/app/controlplane/controlplane_test.go @@ -3,6 +3,7 @@ package controlplane import ( "context" "encoding/json" + "errors" "testing" "time" @@ -74,7 +75,7 @@ func TestImportConfigSeedsWorkflowDefinitionsWithoutOverwritingOverride(t *testi resources := store.OpenTest(t) defer resources.Close() server := testServer(t, resources) - if _, err := server.ImportConfig(t.Context(), config.Config{}); err != nil { + if _, _, err := server.ImportConfig(t.Context(), config.Config{}); err != nil { t.Fatal(err) } resource, err := resources.Resource(t.Context(), WorkflowDefinitionsKind) @@ -96,7 +97,7 @@ func TestImportConfigSeedsWorkflowDefinitionsWithoutOverwritingOverride(t *testi if _, err := resources.PutResource(t.Context(), store.ResourceWrite{Kind: WorkflowDefinitionsKind, Value: value, ExpectedVersion: resource.Version, Actor: "test", Source: "test", RequestID: "override"}); err != nil { t.Fatal(err) } - if _, err := server.ImportConfig(t.Context(), config.Config{}); err != nil { + if _, _, err := server.ImportConfig(t.Context(), config.Config{}); err != nil { t.Fatal(err) } got, err := resources.Resource(t.Context(), WorkflowDefinitionsKind) @@ -114,7 +115,7 @@ func TestImportConfigSeedsPersonasWithoutOverwritingEdits(t *testing.T) { resources := store.OpenTest(t) defer resources.Close() server := testServer(t, resources) - if _, err := server.ImportConfig(t.Context(), config.Config{}); err != nil { + if _, _, err := server.ImportConfig(t.Context(), config.Config{}); err != nil { t.Fatal(err) } resource, err := resources.Resource(t.Context(), PersonasKind) @@ -133,7 +134,7 @@ func TestImportConfigSeedsPersonasWithoutOverwritingEdits(t *testing.T) { if _, err := resources.PutResource(t.Context(), store.ResourceWrite{Kind: PersonasKind, Value: value, ExpectedVersion: resource.Version, Actor: "test", Source: "test", RequestID: "persona-edit"}); err != nil { t.Fatal(err) } - if _, err := server.ImportConfig(t.Context(), config.Config{}); err != nil { + if _, _, err := server.ImportConfig(t.Context(), config.Config{}); err != nil { t.Fatal(err) } got, err := resources.Resource(t.Context(), PersonasKind) @@ -178,9 +179,15 @@ func TestRegistryRoutesAndValidatesDefinitions(t *testing.T) { resources := store.OpenTest(t) defer resources.Close() server := testServer(t, resources) - if _, err := server.ImportConfig(t.Context(), config.Config{Containers: config.ContainerConfig{PullPolicy: "missing"}}); err != nil { + // The image is required, here as in the file document: a seed without one is + // skipped rather than stored, and the Command below would then fail on the + // missing resource instead of on the pull policy it exists to reject. + if _, _, err := server.ImportConfig(t.Context(), config.Config{Containers: config.ContainerConfig{Image: "archie-agent:test", PullPolicy: "missing"}}); err != nil { t.Fatal(err) } + if _, err := resources.Resource(t.Context(), ContainerRuntimePoliciesKind); err != nil { + t.Fatalf("container runtime policies were not seeded: %v", err) + } catalog, err := server.Catalog(t.Context(), &pb.CatalogRequest{}) if err != nil { @@ -209,7 +216,7 @@ func TestProviderSeedKeepsReferencesAndNeverResolvedSecrets(t *testing.T) { defer resources.Close() server := testServer(t, resources) cfg := config.Config{Providers: map[string]config.Provider{"openai": {Class: "openai", APIKeyEnv: "OPENAI_API_KEY", APIKey: config.SecretRef{Engine: "env", Key: "OPENAI_API_KEY"}}}} - if _, err := server.ImportConfig(context.Background(), cfg); err != nil { + if _, _, err := server.ImportConfig(context.Background(), cfg); err != nil { t.Fatal(err) } resource, err := resources.Resource(t.Context(), ProviderSettingsKind) @@ -235,7 +242,7 @@ func TestHistoryCarriesEveryRevisionWithItsAudit(t *testing.T) { resources := store.OpenTest(t) defer resources.Close() server := testServer(t, resources) - if _, err := server.ImportConfig(t.Context(), config.Config{}); err != nil { + if _, _, err := server.ImportConfig(t.Context(), config.Config{}); err != nil { t.Fatal(err) } seeded, err := server.Query(t.Context(), &pb.QueryRequest{Kind: WorkflowExecutionSettingsKind}) @@ -269,3 +276,43 @@ func TestHistoryCarriesEveryRevisionWithItsAudit(t *testing.T) { t.Fatalf("history for an unknown kind = %v, want NotFound", err) } } + +// TestImportConfigSkipsASeedItCannotValidate is the reason a bad seed is not +// fatal. docs/prds/runtime-control-plane.md, "Bootstrap, migration, and +// recovery": after migration, settings in TOML are ignored and cannot block +// State Store startup -- and ImportConfig is on that startup path. The value is +// not stored either, so the setting stays file-owned: the file's value is still +// the one in effect, and editing config.toml is still the fix. Writing the +// invalid value instead would hand ownership to the database and leave the +// operator with a file edit that does nothing. +func TestImportConfigSkipsASeedItCannotValidate(t *testing.T) { + t.Parallel() + + resources := store.OpenTest(t) + defer resources.Close() + server := testServer(t, resources) + // Every other field is filled in the way the loader fills it (Validate + // documents that it does not apply defaults), so the malformed glob is the + // only seed that can be refused. + cfg := validConfigForValidation() + cfg.Repos = []config.Repo{{Owner: "acme", Name: "app", TestGlob: "["}} + + versions, skipped, err := server.ImportConfig(t.Context(), cfg) + if err != nil { + t.Fatalf("ImportConfig: %v, want a skipped seed rather than a fatal one", err) + } + if _, stored := versions[RepositoryPoliciesKind]; stored { + t.Error("the resource was seeded, want the invalid seed skipped") + } + if len(skipped) != 1 || skipped[0].Kind != RepositoryPoliciesKind || skipped[0].Err == nil { + t.Fatalf("skipped = %+v, want the repository policies and the reason", skipped) + } + if _, err := resources.Resource(t.Context(), RepositoryPoliciesKind); !errors.Is(err, store.ErrResourceNotFound) { + t.Errorf("repository policies Resource = %v, want ErrResourceNotFound: an invalid seed must not be stored", err) + } + // Kinds that are fine are still seeded: one skipped kind does not abandon + // the rest of the migration. + if _, err := resources.Resource(t.Context(), WorkflowDefinitionsKind); err != nil { + t.Errorf("workflow definitions: %v, want them seeded alongside the skipped kind", err) + } +} diff --git a/internal/app/controlplane/operational_settings.go b/internal/app/controlplane/operational_settings.go index 1d33d197e..03b9d4591 100644 --- a/internal/app/controlplane/operational_settings.go +++ b/internal/app/controlplane/operational_settings.go @@ -6,6 +6,7 @@ import ( "time" "github.com/samcharles93/archie-core/internal/config" + "github.com/samcharles93/archie-core/internal/infrastructure/configuration" ) const ( @@ -56,6 +57,14 @@ func validateRepositories(input []byte) error { return fmt.Errorf("duplicate repository %q", repo.FullName()) } seen[repo.FullName()] = struct{}{} + // The repository-policies resource replaces the file's list, so the + // rules that judge the file's list have to hold here too. They come + // from the configuration package rather than being restated: a + // resource that accepts what configuration.Validate rejects becomes a + // stored value the operator cannot clear by editing config.toml. + if err := configuration.ValidateTestGlob(repo.ResolvedTestGlob()); err != nil { + return err + } } return nil }) @@ -72,6 +81,12 @@ func validateScheduling(input []byte) error { return fmt.Errorf("poll_interval must be a non-negative duration") } } + // Same reason as the repository glob above: RuntimeConfig replaces + // cfg.Dispatch from this resource, and the daemon refuses to start with a + // trigger it does not know. + if !configuration.DispatchTriggerValid(policy.Dispatch.Trigger) { + return fmt.Errorf("dispatch.trigger %q is not a discovery rule the daemon can poll with", policy.Dispatch.Trigger) + } return nil }) } @@ -84,6 +99,11 @@ func validateContainers(input []byte) error { if settings.PullPolicy != "" && settings.PullPolicy != "missing" && settings.PullPolicy != "always" { return fmt.Errorf("pull_policy must be missing or always") } + // The image is required for autonomous workflow workers; the same rule + // holds for the stored policies that replace the file's [containers]. + if strings.TrimSpace(settings.Image) == "" { + return fmt.Errorf("container image is required for autonomous workflow workers") + } return nil }) } diff --git a/internal/app/controlplane/registry.go b/internal/app/controlplane/registry.go index 629421c93..b7ecf168a 100644 --- a/internal/app/controlplane/registry.go +++ b/internal/app/controlplane/registry.go @@ -61,8 +61,28 @@ func (d Definition) Decode(input []byte) ([]byte, error) { return encoded, nil } -func (s *Server) ImportConfig(ctx context.Context, cfg config.Config) (map[string]int64, error) { +// SeedSkip records a resource ImportConfig could not seed from the file config, +// and why. It is reported rather than returned as an error: see ImportConfig. +type SeedSkip struct { + Kind string + Err error +} + +// ImportConfig seeds every control-plane resource that has no stored value yet +// from the file config, and reports the version of every kind it left alone. +// +// A seed the resource's own validator refuses is SKIPPED and returned in skipped, +// not fatal. The setting it would have carried stays file-owned, so the value in +// effect is still the file's, and the fix is still the file's. +// docs/prds/runtime-control-plane.md, "Bootstrap, migration, and recovery": +// after migration, settings in TOML are ignored and cannot block State Store +// startup -- and this import runs on the State Store's startup path. Fail closed +// belongs to the process that uses the value: boot.runtimeConfig validates the +// effective document and refuses to start archied with a value it cannot run +// with, which is what keeps an invalid value from taking effect silently. +func (s *Server) ImportConfig(ctx context.Context, cfg config.Config) (map[string]int64, []SeedSkip, error) { versions := make(map[string]int64, len(s.ordered)) + var skipped []SeedSkip for _, definition := range s.ordered { resource, err := s.store.Resource(ctx, definition.Kind) if err == nil { @@ -70,19 +90,27 @@ func (s *Server) ImportConfig(ctx context.Context, cfg config.Config) (map[strin continue } if !errors.Is(err, store.ErrResourceNotFound) { - return nil, err + return nil, nil, err } value, err := definition.seededValue(cfg) if err != nil { - return nil, err + // Skipped and reported, not fatal: nothing is written, so the kind + // stays absent and the file document's value is the one in effect, + // while the process that would RUN the value still fails closed on + // it (boot.runtimeConfig validates the effective document). + // seededValue wraps a seed's encode and validation failures together, + // which is why both take this path: the alternative is a second seed + // implementation here to tell them apart. + skipped = append(skipped, SeedSkip{Kind: definition.Kind, Err: err}) + continue } resource, err = s.store.PutResource(ctx, store.ResourceWrite{Kind: definition.Kind, Value: value, Actor: "system:migration", Source: "legacy-config", RequestID: "import:" + definition.Kind, ExpectedVersion: 0, At: time.Now().UTC()}) if err != nil { - return nil, fmt.Errorf("seed %s: %w", definition.Kind, err) + return nil, nil, fmt.Errorf("seed %s: %w", definition.Kind, err) } versions[definition.Kind] = resource.Version } - return versions, nil + return versions, skipped, nil } // seededValue is the value the State Store writes for a kind it does not hold diff --git a/internal/app/controlplane/validators_agree_test.go b/internal/app/controlplane/validators_agree_test.go new file mode 100644 index 000000000..814a058ed --- /dev/null +++ b/internal/app/controlplane/validators_agree_test.go @@ -0,0 +1,150 @@ +package controlplane + +import ( + "encoding/json" + "testing" + "time" + + "github.com/samcharles93/archie-core/internal/config" + "github.com/samcharles93/archie-core/internal/infrastructure/configuration" +) + +// TestResourceValidatorsRejectWhatEffectiveValidationRejects ties the two rule +// sets that guard the same fields. A control-plane resource is seeded from the +// file document and then replaces it wholesale (Client.RuntimeConfig), so a +// value the file path accepts must not be storable through the resource, and a +// value the resource accepts must not stop the daemon's validation of the +// effective document. Guards that live only on one side are how a stale TOML +// value becomes a database value the operator cannot fix by editing config.toml. +// +// Each case mutates the same field the resource carries, so the two halves +// cannot drift apart without failing here. +func TestResourceValidatorsRejectWhatEffectiveValidationRejects(t *testing.T) { + tests := []struct { + name string + kind string + value any + mutate func(cfg *config.Config) + wantErr bool + }{ + { + name: "valid scheduling policy", + kind: SchedulingPolicyKind, + value: map[string]any{"poll_interval": "1m0s", "max_retries": 0, "dispatch": map[string]any{"trigger": "assignee"}}, + mutate: func(cfg *config.Config) {}, + wantErr: false, + }, + { + name: "unrecognised dispatch trigger", + kind: SchedulingPolicyKind, + value: map[string]any{"poll_interval": "1m0s", "max_retries": 0, "dispatch": map[string]any{"trigger": "labels"}}, + mutate: func(cfg *config.Config) { cfg.Dispatch.Trigger = "labels" }, + wantErr: true, + }, + { + name: "negative poll interval", + kind: SchedulingPolicyKind, + value: map[string]any{"poll_interval": "-5s", "max_retries": 0, "dispatch": map[string]any{"trigger": "assignee"}}, + mutate: func(cfg *config.Config) { cfg.PollInterval = config.Duration(-5 * time.Second) }, + wantErr: true, + }, + { + name: "valid repository policies", + kind: RepositoryPoliciesKind, + value: []config.Repo{{Owner: "acme", Name: "app", TestGlob: "**/*_test.go"}}, + mutate: func(cfg *config.Config) {}, + wantErr: false, + }, + { + name: "malformed test glob", + kind: RepositoryPoliciesKind, + value: []config.Repo{{Owner: "acme", Name: "app", TestGlob: "["}}, + mutate: func(cfg *config.Config) { cfg.Repos = []config.Repo{{Owner: "acme", Name: "app", TestGlob: "["}} }, + wantErr: true, + }, + { + name: "valid container runtime policies", + kind: ContainerRuntimePoliciesKind, + value: config.ContainerConfig{Image: "archie-agent:test", PullPolicy: "missing"}, + mutate: func(cfg *config.Config) {}, + wantErr: false, + }, + { + name: "container runtime policies without an image", + kind: ContainerRuntimePoliciesKind, + value: config.ContainerConfig{PullPolicy: "missing"}, + mutate: func(cfg *config.Config) { cfg.Containers.Image = "" }, + wantErr: true, + }, + { + name: "negative container volume ttl", + kind: ContainerRuntimePoliciesKind, + value: config.ContainerConfig{Image: "archie-agent:test", VolumeTTL: config.Duration(-time.Minute)}, + mutate: func(cfg *config.Config) { cfg.Containers.VolumeTTL = config.Duration(-time.Minute) }, + wantErr: true, + }, + { + name: "provider base URL carrying userinfo", + kind: ProviderSettingsKind, + value: map[string]any{"acme": map[string]any{"class": "openai", "base_url": "https://user:pass@example.com"}}, + mutate: func(cfg *config.Config) { + cfg.Providers = map[string]config.Provider{"acme": {Class: "openai", BaseURL: "https://user:pass@example.com"}} + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := validConfigForValidation() + tt.mutate(&cfg) + configErr := configuration.Validate(&cfg) + + encoded, err := json.Marshal(tt.value) + if err != nil { + t.Fatal(err) + } + resourceErr := decodeResource(t, tt.kind, encoded) + + if !tt.wantErr { + if configErr != nil { + t.Errorf("configuration.Validate = %v, want nil for a valid value", configErr) + } + if resourceErr != nil { + t.Errorf("%s Decode = %v, want nil for a valid value", tt.kind, resourceErr) + } + return + } + if configErr == nil { + t.Errorf("configuration.Validate accepted the value, want the effective document to reject it") + } + if resourceErr == nil { + t.Errorf("%s accepted the value, want the resource validator to reject what the effective document rejects", tt.kind) + } + }) + } +} + +// decodeResource runs the resource validator exactly as ImportConfig does when +// it seeds a kind from the file config (registry.go), against the production +// server the composition roots build (testServer). +func decodeResource(t *testing.T, kind string, encoded []byte) error { + t.Helper() + _, err := testServer(t, nil).definitions[kind].Decode(encoded) + return err +} + +// validConfigForValidation is a document every check passes, so each case above +// fails only for the value it changes -- same fixture the configuration package +// uses, restated here because Validate documents that it does not apply +// defaults. +func validConfigForValidation() config.Config { + return config.Config{ + BotUser: "archie-bot", + PollInterval: config.Duration(60 * time.Second), + Dispatch: config.Dispatch{Trigger: "assignee"}, + Forge: config.Forge{Type: "none"}, + Containers: config.ContainerConfig{Image: "ghcr.io/samcharles93/archie-agent:latest"}, + Services: config.Services{config.ServiceNameGateway: {Target: "127.0.0.1:8585"}}, + } +} diff --git a/internal/infrastructure/configuration/apply.go b/internal/infrastructure/configuration/apply.go index fa39ffe49..d7d900151 100644 --- a/internal/infrastructure/configuration/apply.go +++ b/internal/infrastructure/configuration/apply.go @@ -30,23 +30,30 @@ func ApplyOverlayValues(cfg *config.Config, overrides map[string]any) error { return applyOverlayMapping(cfg, overrides) } -// applyOverlayFile layers the overlay file at path over cfg, with the same -// field-level precedence ApplyOverlayValues gives a map overlay. The file -// overlay path needs it for the reason recorded on foldOverrides: decoding an -// overlay into cfg replaces a map-valued entry wholesale, so a file naming -// only services.state.target cleared the target_token the base file set. Both -// overlay paths go through the fold, so neither can drift from the other. -func applyOverlayFile(path string, cfg *config.Config) error { - doc, err := decodeFileMapping(path) +// applyOverlayFile layers the overlay file at path over target, with the same +// field-level precedence ApplyOverlayValues gives a map overlay, and reports the +// file's keys that nothing consumes. target is whatever the overlay addresses: +// the whole config for a main file, one sub-struct for a feature file. +// +// Both overlay forms go through it -- the single overlay file, and every file an +// overlay directory contributes -- so neither form can lose the fields of a +// map-valued entry it only partly addresses. Decoding an overlay into target +// directly replaces such an entry wholesale, which is how an overlay naming only +// services.state.target cleared the target_token the base file set. +func applyOverlayFile(path string, target any) ([]string, error) { + mapping, err := decodeFileMapping(path) if err != nil { - return err + return nil, err } - return applyOverlayMapping(cfg, doc) + if err := applyOverlayMapping(target, mapping); err != nil { + return nil, err + } + return overlayFileKeys(path, mapping, structTypeOf(target)) } // applyOverlayMapping is the decode both overlay paths share: the fields each // map-valued entry the overlay does not name are folded forward out of the -// entry cfg already holds, and the completed mapping is then decoded over cfg. +// entry target already holds, and the completed mapping is then decoded over it. // // yaml gives a struct field the field-level treatment ApplyOverlayValues // promises, but a map VALUE is decoded into a fresh zero value: an entry the @@ -55,8 +62,8 @@ func applyOverlayFile(path string, cfg *config.Config) error { // target_token. Folding here, once, is what makes the promise hold for every // map of structs (services, image.hosted, image.local, providers) instead of // at each field's reader. -func applyOverlayMapping(cfg *config.Config, doc map[string]any) error { - folded, err := foldOverrides(reflect.ValueOf(cfg), doc) +func applyOverlayMapping(target any, doc map[string]any) error { + folded, err := foldOverrides(reflect.ValueOf(target), doc) if err != nil { return err } @@ -64,12 +71,22 @@ func applyOverlayMapping(cfg *config.Config, doc map[string]any) error { if err != nil { return fmt.Errorf("%w: encoding config overlay: %w", ErrUnreadable, err) } - if err := yaml.Unmarshal(data, cfg); err != nil { + if err := yaml.Unmarshal(data, target); err != nil { return fmt.Errorf("%w: parsing config overlay: %w", ErrUnreadable, err) } return nil } +// structTypeOf returns the struct type target points at, so the fold and the key +// report can inspect it without an instance. +func structTypeOf(target any) reflect.Type { + t := reflect.TypeOf(target) + for t != nil && t.Kind() == reflect.Pointer { + t = t.Elem() + } + return t +} + // foldOverrides returns doc with the fields its map entries omit carried // forward from the entries cfg already holds. It descends through structs, so // {"image": {"hosted": {...}}} folds the same way as an override written at the @@ -182,7 +199,16 @@ func mergeMapping(base, override map[string]any) map[string]any { // yamlField returns the field of struct v that the decode reads key into. The // yaml tag is the one consulted because the decode below is yaml's. func yamlField(v reflect.Value, key string) (reflect.Value, bool) { - t := v.Type() + field, ok := yamlFieldOf(v.Type(), key) + if !ok { + return reflect.Value{}, false + } + return v.FieldByIndex(field.Index), true +} + +// yamlFieldOf is yamlField without a value: the overlay key report walks a raw +// mapping against the target's type before anything is decoded into it. +func yamlFieldOf(t reflect.Type, key string) (reflect.StructField, bool) { for i := range t.NumField() { field := t.Field(i) if field.PkgPath != "" { @@ -196,8 +222,8 @@ func yamlField(v reflect.Value, key string) (reflect.Value, bool) { name = strings.ToLower(field.Name) } if name == key { - return v.Field(i), true + return field, true } } - return reflect.Value{}, false + return reflect.StructField{}, false } diff --git a/internal/infrastructure/configuration/decode.go b/internal/infrastructure/configuration/decode.go index 08975ef44..585234efe 100644 --- a/internal/infrastructure/configuration/decode.go +++ b/internal/infrastructure/configuration/decode.go @@ -31,7 +31,7 @@ func decodeConfigFileKeys(path string, target any) ([]string, error) { // carries, with no typed target. An overlay needs this shape: a typed decode // cannot distinguish a key the file omits from a key it sets to the zero // value, and that distinction is what carrying the fields of a partially -// addressed map entry forward is built on (see applyFileOverlay). +// addressed map entry forward is built on (see applyOverlayFile). func decodeFileMapping(path string) (map[string]any, error) { var doc map[string]any switch filepath.Ext(path) { @@ -98,15 +98,25 @@ func decodeYAML(path string, target any) error { // into cfg directly. Memory and tools are sub-structs whose keys sit at the // top level of their own file rather than nested under a "memory:" or // "tools:" key, so they decode into the sub-struct. -func decodeFeature(cfg *config.Config, feature Feature, path string) error { +// +// An overlay layer's feature file is folded over the base layer, for the same +// reason and with the same precedence as a main overlay file. The keys that fold +// reports are dropped here: only a main file reports unknown keys today +// (decodeMain), and a feature file reporting a second, differently-shaped set +// would be a new surface rather than a fix to this one. +func decodeFeature(cfg *config.Config, feature Feature, path string, layer Layer) error { + target := any(cfg) switch feature { case FeatureMemory: - return decodeYAML(path, &cfg.Memory) + target = &cfg.Memory case FeatureTools: - return decodeYAML(path, &cfg.Tools) - default: - return decodeYAML(path, cfg) + target = &cfg.Tools } + if layer == LayerOverlay { + _, err := applyOverlayFile(path, target) + return err + } + return decodeYAML(path, target) } // decodeExtra decodes an unrecognised conf.d/ file into cfg.Extra under name. diff --git a/internal/infrastructure/configuration/loader.go b/internal/infrastructure/configuration/loader.go index beb64085a..5cff2a138 100644 --- a/internal/infrastructure/configuration/loader.go +++ b/internal/infrastructure/configuration/loader.go @@ -5,6 +5,7 @@ import ( "log/slog" "os" "path/filepath" + "reflect" "sort" "github.com/samcharles93/archie-core/internal/config" @@ -175,11 +176,8 @@ func (l *Loader) overlayFile(basePath, overlayPath string) (*Document, error) { if overlayPath != "" { // The overlay is folded over the base config rather than decoded into // it: a decode replaces a map-valued entry wholesale, clearing the - // fields of that entry the overlay does not name (applyFileOverlay). - if err := applyOverlayFile(overlayPath, &doc.Config); err != nil { - return nil, err - } - cfgKeys, err = overlayFileKeys(overlayPath) + // fields of that entry the overlay does not name (applyOverlayFile). + cfgKeys, err = applyOverlayFile(overlayPath, &doc.Config) if err != nil { return nil, err } @@ -194,13 +192,69 @@ func (l *Loader) overlayFile(basePath, overlayPath string) (*Document, error) { return l.finalize(doc, validateBootstrap) } -// overlayFileKeys reports the top-level keys of an overlay file that the -// config decode does not consume. The file itself reaches the config through -// applyOverlayFile, which needs its raw mapping; this typed decode exists only -// for the key report, which only TOML's decoder can produce. -func overlayFileKeys(path string) ([]string, error) { - var scratch config.Config - return decodeConfigFileKeys(path, &scratch) +// overlayFileKeys reports the keys of an overlay file that nothing consumes. It +// has two sources, because the file's own decode and the fold's apply decode do +// not match keys the same way: TOML accepts a key that differs only in case, +// while the yaml decode the fold uses does not. A key only the file's decode +// matches would therefore be consumed by the report below and dropped in +// silence by the apply, so the apply's own view of the mapping is unioned in. +// +// The second decode here is deliberate: it is what carries TOML's Undecoded() +// view of the file, which the raw mapping cannot report (it consumes every key). +func overlayFileKeys(path string, mapping map[string]any, target reflect.Type) ([]string, error) { + decoded, err := decodeConfigFileKeys(path, reflect.New(target).Interface()) + if err != nil { + return nil, err + } + return append(decoded, unmatchableKeys(mapping, target)...), nil +} + +// unmatchableKeys returns the dotted paths of the keys in mapping that the fold's +// yaml decode cannot match to a field of target. It descends the way foldOverrides +// does -- through structs and through string-keyed maps of structs -- so it sees +// the same keys the apply does. A slice of tables is not descended: the yaml +// decode replaces it wholesale, so anything inside it is a value, not a key. +func unmatchableKeys(mapping map[string]any, target reflect.Type) []string { + var out []string + for key, value := range mapping { + field, ok := yamlFieldOf(target, key) + if !ok { + out = append(out, key) + continue + } + sub, ok := value.(map[string]any) + if !ok { + continue + } + switch field.Type.Kind() { + case reflect.Struct: + out = append(out, prefixed(key, unmatchableKeys(sub, field.Type))...) + case reflect.Map: + entry := field.Type.Elem() + for entry.Kind() == reflect.Pointer { + entry = entry.Elem() + } + if entry.Kind() != reflect.Struct || field.Type.Key().Kind() != reflect.String { + continue + } + for entryName, entryValue := range sub { + entryMapping, ok := entryValue.(map[string]any) + if !ok { + continue + } + out = append(out, prefixed(key+"."+entryName, unmatchableKeys(entryMapping, entry))...) + } + } + } + return out +} + +// prefixed qualifies a nested key report with the path that reached it. +func prefixed(prefix string, keys []string) []string { + for i, key := range keys { + keys[i] = prefix + "." + key + } + return keys } // unknownKeys returns the keys present in both a and b: one config file @@ -303,7 +357,7 @@ func (l *Loader) loadDir(doc *Document, dir string, layer Layer) error { switch path, isYAMLFile, ok := files.main(); { case ok: - if err := l.decodeMain(doc, path, isYAMLFile); err != nil { + if err := l.decodeMain(doc, path, isYAMLFile, layer); err != nil { return err } doc.Provenance.record(Origin{Path: path, Role: RoleMain, Layer: layer}) @@ -313,7 +367,7 @@ func (l *Loader) loadDir(doc *Document, dir string, layer Layer) error { for _, feature := range files.sortedFeatures() { path := files.features[feature] - if err := decodeFeature(&doc.Config, feature, path); err != nil { + if err := decodeFeature(&doc.Config, feature, path, layer); err != nil { return err } doc.Provenance.record(Origin{Path: path, Role: RoleFeature, Layer: layer, Feature: feature}) @@ -330,8 +384,20 @@ func (l *Loader) loadDir(doc *Document, dir string, layer Layer) error { return nil } -// decodeMain decodes the daemon-level file in whichever format it uses. -func (l *Loader) decodeMain(doc *Document, path string, isYAMLFile bool) error { +// decodeMain decodes the daemon-level file in whichever format it uses. An +// overlay layer's file is FOLDED over what the base layer decoded, the same way +// a single overlay file folds over its base -- otherwise a map-valued entry it +// only partly addresses would clear the fields it does not name, which is the +// archie-core-e2e2 failure on the directory form of the same flag. +func (l *Loader) decodeMain(doc *Document, path string, isYAMLFile bool, layer Layer) error { + if layer == LayerOverlay { + cfgKeys, err := applyOverlayFile(path, &doc.Config) + if err != nil { + return err + } + doc.UnknownKeys = append(doc.UnknownKeys, cfgKeys...) + return decodeSchedulingFile(path, &doc.Scheduling) + } if isYAMLFile { if err := decodeYAML(path, &doc.Config); err != nil { return err diff --git a/internal/infrastructure/configuration/loader_test.go b/internal/infrastructure/configuration/loader_test.go index b8c1a7bdc..f61c14da3 100644 --- a/internal/infrastructure/configuration/loader_test.go +++ b/internal/infrastructure/configuration/loader_test.go @@ -241,10 +241,10 @@ func TestLoadTelegramTokenSecretRefAndLegacyFallback(t *testing.T) { // settings the control plane owns are layered over it later; a stale TOML value // in one of those must not fail the load (docs/prds/runtime-control-plane.md, // "Bootstrap, migration, and recovery"). So a check over a database-owned -// setting is asserted on the effective document instead -- the same document -// boot.runtimeConfig validates after the merge. Each case asserts both halves -// it applies to, so a check that quietly disappeared fails the case rather than -// passing it. +// setting is asserted with Validate instead -- what boot.runtimeConfig runs over +// the document a process will use, and the checks the loader no longer applies. +// Each case asserts both halves it applies to, so a check that quietly +// disappeared fails the case rather than passing it. func TestLoadRejectsInvalidConfigEnumsAndGlobs(t *testing.T) { tests := []struct { name string @@ -309,7 +309,7 @@ func TestLoadRejectsInvalidConfigEnumsAndGlobs(t *testing.T) { t.Fatalf("File: %v (a database-owned value must not fail the load)", err) } if err := Validate(&doc.Config); err == nil { - t.Fatal("Validate(effective document) = nil, want the value rejected there") + t.Fatal("Validate = nil, want the value rejected there") } }) } @@ -537,11 +537,10 @@ func TestLoadContainerVolumeTTL(t *testing.T) { } // TestLoadRejectsNegativeContainerVolumeTTL: containers are a control-plane -// resource (container-runtime-policies), so a negative volume_ttl is judged on -// the effective document rather than on the file the load path reads -- a stale -// TOML value in a database-owned setting must not fail a process's startup. The -// second half of the assertion is what keeps dropping the check from passing -// this test. +// resource (container-runtime-policies), so a negative volume_ttl is judged by +// Validate rather than by the load path -- a stale TOML value in a database-owned +// setting must not fail a process's startup. The second half of the assertion is +// what keeps dropping the check from passing this test. func TestLoadRejectsNegativeContainerVolumeTTL(t *testing.T) { path := filepath.Join(t.TempDir(), "config.toml") contents := "bot_user = \"widget\"\n" + @@ -557,7 +556,7 @@ func TestLoadRejectsNegativeContainerVolumeTTL(t *testing.T) { t.Fatalf("File: %v (a database-owned value must not fail the load)", err) } if err := Validate(&doc.Config); err == nil { - t.Fatal("Validate(effective document) accepted a negative containers.volume_ttl") + t.Fatal("Validate accepted a negative containers.volume_ttl") } } @@ -635,15 +634,16 @@ func TestOverlay(t *testing.T) { // (TestApplyOverlayValuesPreservesOmittedNestedFields). // // It goes through Loader.Resolve, the production entry point, because the -// hazard is a property of how the overlay file is applied, not of any one -// decode helper: Loader.overlayFile decodes the overlay into the very config -// the base file produced, and a map-valued entry is replaced wholesale, so -// every field of that entry the overlay does not name is cleared. +// hazard was a property of how the overlay file is applied, not of any one +// decode helper: Loader.overlayFile decoded the overlay into the very config +// the base file produced, and a map-valued entry is replaced wholesale, so every +// field of that entry the overlay does not name was cleared. // // That is archie-core-e2e2 on a user-visible path. deployments/dev.toml is the // documented -config-overlay argument and sets [services.state] target/listen, // while the installed config.toml is where [services.state].target_token lives: -// the overlay silently drops the token. +// the overlay silently dropped the token. The directory form of the same flag is +// covered by TestDirOverlayPreservesOmittedMapEntryFields. func TestResolveFileOverlayPreservesOmittedMapEntryFields(t *testing.T) { tests := []struct { name string @@ -757,18 +757,21 @@ func TestResolveFileOverlayPreservesOmittedMapEntryFields(t *testing.T) { // TestLoadAcceptsStaleDatabaseOwnedValues is the bootstrap half of the layer // split (archie-core-i3qm). docs/prds/runtime-control-plane.md, // "Bootstrap, migration, and recovery": after migration, settings in TOML are -// ignored and cannot block State Store startup. archie-state-store never layers -// a database resource over its file document -- state_store.go:78 resolves the -// file and goes straight to opening the store -- so a stale value in a setting -// the control plane owns must not fail the load. +// ignored and cannot block State Store startup. The other half is the boot +// itself -- archie-state-store seeds these same settings into the control plane +// on a fresh database, and a seed it cannot validate is skipped rather than fatal +// (TestStateStoreBootsWithStaleDatabaseOwnedValues boots the process for that). // // Each case is a setting with a control-plane resource behind it, seeded from // that field by controlplane.Server.ImportConfig: providers // (provider-settings), poll_interval and dispatch (scheduling-policy), // containers (container-runtime-policies), repositories -// (repository-policies). The second half of the assertion is what keeps this -// from being a weaker gate: the effective document, judged after the control -// plane has layered its own value in, must still reject the value. +// (repository-policies). The second assertion is what keeps this from being a +// weaker gate: the value is still rejected by Validate, which is what the daemon +// and the Gateway run over the document a process will actually use. Nothing in +// this package layers that document, so the check here is the file document on +// its own -- the test is that the loader judges no database-owned field, not +// that the value is harmless. func TestLoadAcceptsStaleDatabaseOwnedValues(t *testing.T) { tests := []struct { name string @@ -808,7 +811,7 @@ func TestLoadAcceptsStaleDatabaseOwnedValues(t *testing.T) { t.Fatalf("File: %v (a stale database-owned value must not fail the load)", err) } if err := Validate(&doc.Config); err == nil { - t.Fatal("Validate(effective document) = nil, want the control plane's own layer to reject it") + t.Fatal("Validate = nil, want the control plane's own layer to reject it") } }) } @@ -818,6 +821,82 @@ func TestLoadAcceptsStaleDatabaseOwnedValues(t *testing.T) { // so each case above adds one stale setting and nothing else. const fileConfigPrefix = "bot_user = \"widget\"\nwork_dir = \"/base/work\"\n" +// TestDirOverlayPreservesOmittedMapEntryFields is the directory form of the +// same guarantee TestResolveFileOverlayPreservesOmittedMapEntryFields pins. +// -config-overlay accepts a directory as well as a file, Loader.Dir documents +// "the same field-level precedence as [Loader.Overlay]", and Loader.Resolve +// routes a directory source to Dir -- so a map-valued entry the overlay only +// partly addresses must keep the fields it does not name on both forms of the +// flag, including when the entry arrives through a feature file. +func TestDirOverlayPreservesOmittedMapEntryFields(t *testing.T) { + tests := []struct { + name string + baseFiles map[string]string + overlayFiles map[string]string + check func(t *testing.T, cfg config.Config) + }{ + { + name: "main config keeps the token it does not name", + baseFiles: map[string]string{ + "config.yaml": "bot_user: widget\nservices:\n state:\n target: 127.0.0.1:9090\n target_token: secret\n", + }, + overlayFiles: map[string]string{ + "config.yaml": "services:\n state:\n target: 10.0.0.5:9090\n", + }, + check: func(t *testing.T, cfg config.Config) { + t.Helper() + got := cfg.Services[config.ServiceNameState] + if got.Target != "10.0.0.5:9090" { + t.Errorf("services.state.target = %q, want the overlay's 10.0.0.5:9090", got.Target) + } + if got.TargetToken != "secret" { + t.Errorf("services.state.target_token = %q, want the base's secret: a directory overlay must not clear a field it does not name", got.TargetToken) + } + }, + }, + { + name: "feature file keeps the fields it does not name", + baseFiles: map[string]string{ + "config.yaml": "bot_user: widget\n", + "config.models.yaml": "providers:\n anthropic:\n class: anthropic\n api_key_env: ANTHROPIC_API_KEY\n", + }, + overlayFiles: map[string]string{ + "config.models.yaml": "providers:\n anthropic:\n base_url: https://proxy.example\n", + }, + check: func(t *testing.T, cfg config.Config) { + t.Helper() + got := cfg.Providers["anthropic"] + want := config.Provider{ + Class: "anthropic", + APIKeyEnv: "ANTHROPIC_API_KEY", + BaseURL: "https://proxy.example", + } + if got != want { + t.Errorf("providers.anthropic = %+v, want %+v", got, want) + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + baseDir, overlayDir := tmpConfigDir(t), tmpConfigDir(t) + for name, content := range tt.baseFiles { + writeFile(t, baseDir, name, content) + } + for name, content := range tt.overlayFiles { + writeFile(t, overlayDir, name, content) + } + + doc, err := New(nil).Resolve(baseDir, overlayDir) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + tt.check(t, doc.Config) + }) + } +} + func TestLoadBytesDoesNotExist(t *testing.T) { _, err := loadBytes(nil) if err == nil { diff --git a/internal/infrastructure/configuration/unknown_keys_test.go b/internal/infrastructure/configuration/unknown_keys_test.go index 7896e664c..48a3b7d65 100644 --- a/internal/infrastructure/configuration/unknown_keys_test.go +++ b/internal/infrastructure/configuration/unknown_keys_test.go @@ -91,28 +91,55 @@ func TestUnknownSchedulingKeyIsStillReported(t *testing.T) { // TestOverlayUnknownKeysAreStillReported pins that the overlay path kept its // drift detection when the overlay file started being applied through the -// config fold (applyOverlayFile) instead of being decoded into the document's -// config: the key report comes from a separate decode, so losing it would be -// silent. +// config fold instead of being decoded into the document's config. That matters +// twice over there: the report comes from a separate decode, and a file is +// parsed by TOML while the fold applies it with a yaml decode, which do not +// match keys the same way. A key the APPLY cannot consume has to be reported, +// or a spelling that works in the base config would silently do nothing in the +// overlay. func TestOverlayUnknownKeysAreStillReported(t *testing.T) { - dir := t.TempDir() - basePath := filepath.Join(dir, "config.toml") - overlayPath := filepath.Join(dir, "dev.toml") - if err := os.WriteFile(basePath, []byte(minimalValidConfigTOML), 0o600); err != nil { - t.Fatal(err) - } - // typo: concurrancy. Unknown to both decode targets, so it is a real - // unknown key rather than Hazard 1's cross-target case. - if err := os.WriteFile(overlayPath, []byte("[containers]\nmax_concurrancy = 4\n"), 0o600); err != nil { - t.Fatal(err) + tests := []struct { + name string + overlay string + wantUnknown string + }{ + { + name: "a key no decode target consumes", + overlay: "[containers]\nmax_concurrancy = 4\n", // typo: concurrancy + wantUnknown: "containers.max_concurrancy", + }, + { + // TOML matches keys case-insensitively, so the file's own decode + // consumes BOT_USER and reports nothing; the apply folds the file's + // mapping with a yaml decode, which does not match it. Without the + // second source for this report the value would be dropped in + // silence, so the spelling is reported instead. + name: "a key only the file decode can match", + overlay: "BOT_USER = \"upper\"\n", + wantUnknown: "BOT_USER", + }, } - doc, err := New(nil).Resolve(basePath, overlayPath) - if err != nil { - t.Fatalf("Resolve: %v (an unknown key must not fail the load)", err) - } - if !slices.Contains(doc.UnknownKeys, "containers.max_concurrancy") { - t.Errorf("UnknownKeys = %v, want it to contain %q", doc.UnknownKeys, "containers.max_concurrancy") + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + basePath := filepath.Join(dir, "config.toml") + overlayPath := filepath.Join(dir, "dev.toml") + if err := os.WriteFile(basePath, []byte(minimalValidConfigTOML), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(overlayPath, []byte(tt.overlay), 0o600); err != nil { + t.Fatal(err) + } + + doc, err := New(nil).Resolve(basePath, overlayPath) + if err != nil { + t.Fatalf("Resolve: %v (an unknown key must not fail the load)", err) + } + if !slices.Contains(doc.UnknownKeys, tt.wantUnknown) { + t.Errorf("UnknownKeys = %v, want it to contain %q", doc.UnknownKeys, tt.wantUnknown) + } + }) } } diff --git a/internal/infrastructure/configuration/validate.go b/internal/infrastructure/configuration/validate.go index 61d6eef61..557c3a808 100644 --- a/internal/infrastructure/configuration/validate.go +++ b/internal/infrastructure/configuration/validate.go @@ -86,10 +86,13 @@ func validate(cfg *config.Config) error { // TOML value in one of them must not be able to fail a process's startup // (docs/prds/runtime-control-plane.md, "Bootstrap, migration, and recovery": // after migration, settings in TOML are ignored and cannot block State Store -// startup). archie-state-store resolves its file config and opens the store -// without ever layering a database resource over it (state_store.go), so a -// check left here is a check that can still block it. Those checks live in -// validateDatabaseOwned and run on the effective document instead. +// startup). archie-state-store resolves its file config and seeds the +// control-plane resources from that same document on a fresh database, and a +// seed it cannot validate is skipped rather than fatal +// (controlplane.Server.ImportConfig), so a check left here is a check that can +// still block it. Those checks live in validateDatabaseOwned and run on the +// effective document instead -- which is where the process that uses the value +// refuses it, and where the operator's fix is the file again. func validateBootstrap(cfg *config.Config) error { if err := validateForgeIntake(cfg); err != nil { return err @@ -227,7 +230,7 @@ func validatePollInterval(cfg *config.Config) error { // labels) was configured while the actual trigger-match label was left // blank. func validateDispatch(cfg *config.Config) error { - if !oneOf(cfg.Dispatch.Trigger, dispatchTriggers) { + if !DispatchTriggerValid(cfg.Dispatch.Trigger) { return fmt.Errorf("%w: dispatch.trigger %q (want %s)", ErrInvalidInput, cfg.Dispatch.Trigger, list(dispatchTriggers)) } if (cfg.Dispatch.Trigger == dispatchTriggerLabel || cfg.Dispatch.Trigger == dispatchTriggerEither) && cfg.Label == "" { @@ -236,6 +239,16 @@ func validateDispatch(cfg *config.Config) error { return nil } +// DispatchTriggerValid reports whether trigger names a discovery rule the daemon +// can poll with. It is exported because the scheduling-policy resource validator +// (internal/app/controlplane) judges the same field: that resource is seeded +// from cfg.Dispatch and replaces it wholesale once the database owns it, so a +// value this package rejects must not be storable, and a value it accepts must +// not be rejected there. One definition, both layers. +func DispatchTriggerValid(trigger string) bool { + return oneOf(trigger, dispatchTriggers) +} + // validateForgeIntake checks the forge intake mode and that webhook intake has // the secret and listen address it needs. An unset intake resolves to "poll" // without mutating cfg, so Validate (which does not apply defaults) accepts a @@ -332,9 +345,12 @@ func validateSingleIdentity(cfg *config.Config) error { } // validateRepositoryContents judges the repositories themselves, whichever list -// they appear in. Repository policies are a control-plane resource seeded from -// these, so the contents are verified on the effective document -- the one the -// database's list has already replaced the file's copy in. +// they appear in. The shared list is the decision the database's copy replaces, +// so it is verified on the effective document -- after that replacement, which is +// the only point the value being judged is the one a process will use. The +// per-identity lists are verified there for a different reason: no resource +// carries them, and the daemon is the only process that reads them +// (docs/architecture/configuration.md's IdentityConfig note). func validateRepositoryContents(cfg *config.Config) error { if len(cfg.Identities) == 0 { return validateRepos(cfg.Repos) @@ -352,15 +368,28 @@ func validateRepos(repos []config.Repo) error { if r.Owner == "" || r.Name == "" { return fmt.Errorf("%w: repos[%d] needs owner and name", ErrInvalidInput, i) } - if glob := r.ResolvedTestGlob(); glob != "" { - if _, err := filepath.Match(glob, ""); err != nil { - return fmt.Errorf("%w: repos[%d] test_glob %q: %w", ErrInvalidInput, i, glob, err) - } + if err := ValidateTestGlob(r.ResolvedTestGlob()); err != nil { + return fmt.Errorf("%w: repos[%d] %w", ErrInvalidInput, i, err) } } return nil } +// ValidateTestGlob reports whether glob is a pattern the test-protection gate can +// compile. It is exported because the repository-policies resource validator +// (internal/app/controlplane) judges the same field: that resource is seeded from +// the file's repositories and replaces them wholesale, and a pattern the gate +// cannot compile is a value that must not be storable either. +func ValidateTestGlob(glob string) error { + if glob == "" { + return nil + } + if _, err := filepath.Match(glob, ""); err != nil { + return fmt.Errorf("test_glob %q: %w", glob, err) + } + return nil +} + // effectiveNATSMode resolves an unset nats.mode from url without mutating cfg, // so validation that reads the mode never has to know whether the operator // spelled it or left it to default. validateNATS and validateContainers both