From 369dfdf550235486c7e238f8748de8d5197c0b10 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Sat, 19 Sep 2026 03:07:14 +1000 Subject: [PATCH 1/4] fix(config): stop a partial overlay clearing the fields it omits An overlay or dashboard PATCH naming only some fields of a map-valued config entry silently zeroed the rest. For [services.] that meant losing the bearer token, so changing a State Store endpoint dropped its authentication: start Services{"state": {Target: "127.0.0.1:50051", TargetToken: "secret"}} overlay {"services": {"state": {"target": "10.0.0.5:50051"}}} result Services{"state": {Target: "10.0.0.5:50051", TargetToken: ""}} Cause: ApplyOverlayValues did yaml.Marshal(overrides) then yaml.Unmarshal into cfg. yaml gives a struct FIELD the field-level treatment the doc comment promises, but a map VALUE is decoded into a fresh zero value, so an entry the overlay only partly addresses loses every field it does not name. That contract was written when Services was a struct; making it map[string]ServiceConnection changed the behaviour underneath it. Fix at the producer: fold the fields the overlay omits forward out of the entry cfg already holds, so the decoder is handed a complete entry. Recursion follows the destination type, so the promise now holds for every map of structs -- services, image.hosted, image.local and providers -- in one place instead of at each field's reader. Struct field-level merge, scalar-valued map-key merge and slice replacement are unchanged, and overrides is read, never mutated. The regression test also converts to the map form, which is what unbreaks the build: apply_test.go still built Services with the struct form after services.go became a map, and because the two live in different files the breakage merged cleanly. There is no Go build or test CI, so nothing compiled it. Red: four cases failed with the token, the target and the sibling fields zeroed. Green after the fold. --- .../infrastructure/configuration/apply.go | 152 +++++++++++++++++- .../configuration/apply_test.go | 91 +++++++++-- 2 files changed, 231 insertions(+), 12 deletions(-) diff --git a/internal/infrastructure/configuration/apply.go b/internal/infrastructure/configuration/apply.go index 3edd5e307..f73402989 100644 --- a/internal/infrastructure/configuration/apply.go +++ b/internal/infrastructure/configuration/apply.go @@ -2,6 +2,9 @@ package configuration import ( "fmt" + "maps" + "reflect" + "strings" "gopkg.in/yaml.v3" @@ -18,11 +21,26 @@ import ( // dashboard PATCH path calls it on a copy of the published config and // validates the materialised result before persisting. Either way the // caller owns the cfg value -- pass a copy when the base must survive. +// +// overrides is read, never mutated. func ApplyOverlayValues(cfg *config.Config, overrides map[string]any) error { if len(overrides) == 0 { return nil } - data, err := yaml.Marshal(overrides) + // yaml gives a struct field the field-level treatment promised above, but a + // map VALUE is decoded into a fresh zero value: an entry the overlay only + // partially addresses would lose every field it does not name, so an + // operator changing services.state.target alone would silently drop + // target_token. Fold the fields the overlay omits forward out of the entry + // cfg already holds, leaving the decode a complete entry to read. Doing it + // here, once, is what makes the promise hold for every map of structs + // (services, image.hosted, image.local, providers) instead of at each + // field's reader. + folded, err := foldOverrides(reflect.ValueOf(cfg), overrides) + if err != nil { + return err + } + data, err := yaml.Marshal(folded) if err != nil { return fmt.Errorf("%w: encoding config overlay: %w", ErrUnreadable, err) } @@ -31,3 +49,135 @@ func ApplyOverlayValues(cfg *config.Config, overrides map[string]any) error { } return nil } + +// foldOverrides returns doc with the fields its map entries omit carried +// forward from the entries cfg already holds. It descends through structs, so +// {"image": {"hosted": {...}}} folds the same way as an override written at the +// top level. +// +// Only mapping-valued map entries need it: yaml keeps the entries a map of +// scalars does not name, and a struct field it does not name keeps its value. +func foldOverrides(cfg reflect.Value, doc map[string]any) (map[string]any, error) { + if cfg.Kind() == reflect.Pointer { + if cfg.IsNil() { + return doc, nil + } + cfg = cfg.Elem() + } + if cfg.Kind() != reflect.Struct { + return doc, nil + } + out := make(map[string]any, len(doc)) + maps.Copy(out, doc) + for key, value := range out { + field, ok := yamlField(cfg, key) + if !ok { + continue + } + sub, ok := value.(map[string]any) + if !ok { + continue // a scalar replaces the whole field, and the decode says so + } + switch field.Kind() { + case reflect.Struct: + folded, err := foldOverrides(field, sub) + if err != nil { + return nil, err + } + out[key] = folded + case reflect.Map: + folded, err := foldMapEntries(field, sub) + if err != nil { + return nil, err + } + out[key] = folded + } + } + return out, nil +} + +// foldMapEntries returns one map field's override with each entry the overlay +// names carried forward from the entry of the same key in cfg. +func foldMapEntries(field reflect.Value, doc map[string]any) (map[string]any, error) { + if field.IsNil() || field.Type().Key().Kind() != reflect.String { + return doc, nil + } + out := make(map[string]any, len(doc)) + maps.Copy(out, doc) + for key, value := range out { + entry, ok := value.(map[string]any) + if !ok { + continue + } + existing := field.MapIndex(reflect.ValueOf(key).Convert(field.Type().Key())) + if !existing.IsValid() { + continue // an entry the overlay introduces has no earlier fields + } + carried, err := asMapping(existing) + if err != nil { + return nil, err + } + if carried == nil { + continue // not a mapping, so the overlay replaces it wholesale + } + out[key] = mergeMapping(carried, entry) + } + return out, nil +} + +// asMapping renders one config value as the mapping the decode will read it +// back as, which is what lets the fields an override omits be carried forward. +// It returns nil when the value does not encode to a mapping. +func asMapping(value reflect.Value) (map[string]any, error) { + data, err := yaml.Marshal(value.Interface()) + if err != nil { + return nil, fmt.Errorf("%w: encoding config overlay: %w", ErrUnreadable, err) + } + // Decoding into any cannot fail for marshalled output; a value that is not + // a mapping simply yields no mapping. + var generic any + _ = yaml.Unmarshal(data, &generic) + mapping, _ := generic.(map[string]any) + return mapping, nil +} + +// mergeMapping layers override over base: mappings merge key by key, anything +// else replaces, matching how the decode treats a struct field versus a scalar. +// It allocates rather than mutating either argument. +func mergeMapping(base, override map[string]any) map[string]any { + out := make(map[string]any, len(base)+len(override)) + maps.Copy(out, base) + for key, value := range override { + if over, ok := value.(map[string]any); ok { + if under, ok := base[key].(map[string]any); ok { + out[key] = mergeMapping(under, over) + continue + } + } + out[key] = value + } + return out +} + +// yamlField returns the field of struct v that the decode reads key into. The +// yaml tag is the one consulted because the decode below is yaml's. +func yamlField(v reflect.Value, key string) (reflect.Value, bool) { + t := v.Type() + for i := range t.NumField() { + field := t.Field(i) + if field.PkgPath != "" { + continue // unexported, so the decoder cannot write it + } + name, _, _ := strings.Cut(field.Tag.Get("yaml"), ",") + if name == "-" { + continue + } + if name == "" { + name = strings.ToLower(field.Name) + } + if name == key { + return v.Field(i), true + } + } + return reflect.Value{}, false +} diff --git a/internal/infrastructure/configuration/apply_test.go b/internal/infrastructure/configuration/apply_test.go index d0b03ebca..467b96824 100644 --- a/internal/infrastructure/configuration/apply_test.go +++ b/internal/infrastructure/configuration/apply_test.go @@ -2,6 +2,7 @@ package configuration import ( "log/slog" + "maps" "os" "path/filepath" "testing" @@ -38,7 +39,7 @@ func TestApplyOverlayValuesFieldLevelMerge(t *testing.T) { // must not clear target_token, because dropping the token silently removes // authentication for a non-loopback State Store. The existing field-level test // only covers one level (budgets.max_steps); these cases go two levels deep into -// a struct of structs, which is the shape Services has. +// a map of structs, which is the shape Services has. func TestApplyOverlayValuesPreservesOmittedNestedFields(t *testing.T) { tests := []struct { name string @@ -48,21 +49,27 @@ func TestApplyOverlayValuesPreservesOmittedNestedFields(t *testing.T) { }{ { name: "target-only overlay keeps the token", - start: config.Services{State: config.ServiceConnection{Target: "127.0.0.1:50051", TargetToken: "secret"}}, - overlay: map[string]any{"services": map[string]any{"state": map[string]any{"target": "10.0.0.5:50051"}}}, - want: config.Services{State: config.ServiceConnection{Target: "10.0.0.5:50051", TargetToken: "secret"}}, + start: config.Services{config.ServiceNameState: {Target: "127.0.0.1:50051", TargetToken: "secret"}}, + overlay: map[string]any{"services": map[string]any{config.ServiceNameState: map[string]any{"target": "10.0.0.5:50051"}}}, + want: config.Services{config.ServiceNameState: {Target: "10.0.0.5:50051", TargetToken: "secret"}}, }, { name: "token-only overlay keeps the target", - start: config.Services{State: config.ServiceConnection{Target: "10.0.0.5:50051", TargetToken: "old"}}, - overlay: map[string]any{"services": map[string]any{"state": map[string]any{"target_token": "rotated"}}}, - want: config.Services{State: config.ServiceConnection{Target: "10.0.0.5:50051", TargetToken: "rotated"}}, + start: config.Services{config.ServiceNameState: {Target: "10.0.0.5:50051", TargetToken: "old"}}, + overlay: map[string]any{"services": map[string]any{config.ServiceNameState: map[string]any{"target_token": "rotated"}}}, + want: config.Services{config.ServiceNameState: {Target: "10.0.0.5:50051", TargetToken: "rotated"}}, }, { name: "sibling service is untouched", - start: config.Services{Gateway: config.ServiceConnection{Target: "127.0.0.1:50052"}, State: config.ServiceConnection{Target: "a", TargetToken: "t"}}, - overlay: map[string]any{"services": map[string]any{"state": map[string]any{"target": "b"}}}, - want: config.Services{Gateway: config.ServiceConnection{Target: "127.0.0.1:50052"}, State: config.ServiceConnection{Target: "b", TargetToken: "t"}}, + start: config.Services{config.ServiceNameGateway: {Target: "127.0.0.1:50052"}, config.ServiceNameState: {Target: "a", TargetToken: "t"}}, + overlay: map[string]any{"services": map[string]any{config.ServiceNameState: map[string]any{"target": "b"}}}, + want: config.Services{config.ServiceNameGateway: {Target: "127.0.0.1:50052"}, config.ServiceNameState: {Target: "b", TargetToken: "t"}}, + }, + { + name: "an entry the overlay introduces decodes in full", + start: config.Services{}, + overlay: map[string]any{"services": map[string]any{config.ServiceNameState: map[string]any{"target": "127.0.0.1:50051", "target_token": "fresh"}}}, + want: config.Services{config.ServiceNameState: {Target: "127.0.0.1:50051", TargetToken: "fresh"}}, }, } @@ -72,13 +79,75 @@ func TestApplyOverlayValuesPreservesOmittedNestedFields(t *testing.T) { if err := ApplyOverlayValues(&cfg, tt.overlay); err != nil { t.Fatalf("ApplyOverlayValues: %v", err) } - if cfg.Services != tt.want { + if !maps.Equal(cfg.Services, tt.want) { t.Errorf("Services = %+v, want %+v", cfg.Services, tt.want) } }) } } +// TestApplyOverlayValuesMergesEveryMapEntry pins that the preservation above is +// a property of ApplyOverlayValues rather than of Services. yaml.v3 decodes a +// map value into a fresh zero value, so every map[string]struct field loses the +// fields an override omits; Services, image.hosted and providers are the three +// the configuration has, so a fix that named one of them would leave the other +// two silently zeroing operator settings. +func TestApplyOverlayValuesMergesEveryMapEntry(t *testing.T) { + tests := []struct { + name string + start config.Config + overlay map[string]any + check func(t *testing.T, cfg config.Config) + }{ + { + name: "image.hosted entry keeps the fields the overlay omits", + start: config.Config{Image: config.ImageConfig{Hosted: map[string]config.ImageHostedProvider{ + "minimax": {Enabled: true, Class: "minimax", APIKeyEnv: "MINIMAX_API_KEY"}, + }}}, + overlay: map[string]any{"image": map[string]any{"hosted": map[string]any{ + "minimax": map[string]any{"base_url": "https://api.example"}, + }}}, + check: func(t *testing.T, cfg config.Config) { + t.Helper() + want := map[string]config.ImageHostedProvider{ + "minimax": {Enabled: true, Class: "minimax", APIKeyEnv: "MINIMAX_API_KEY", BaseURL: "https://api.example"}, + } + if !maps.Equal(cfg.Image.Hosted, want) { + t.Errorf("Image.Hosted = %+v, want %+v", cfg.Image.Hosted, want) + } + }, + }, + { + name: "providers entry keeps the fields the overlay omits", + start: config.Config{Providers: map[string]config.Provider{ + "anthropic": {Class: "anthropic", APIKeyEnv: "ANTHROPIC_API_KEY"}, + }}, + overlay: map[string]any{"providers": map[string]any{ + "anthropic": map[string]any{"base_url": "https://proxy.example"}, + }}, + check: func(t *testing.T, cfg config.Config) { + t.Helper() + want := map[string]config.Provider{ + "anthropic": {Class: "anthropic", APIKeyEnv: "ANTHROPIC_API_KEY", BaseURL: "https://proxy.example"}, + } + if !maps.Equal(cfg.Providers, want) { + t.Errorf("Providers = %+v, want %+v", cfg.Providers, want) + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := tt.start + if err := ApplyOverlayValues(&cfg, tt.overlay); err != nil { + t.Fatalf("ApplyOverlayValues: %v", err) + } + tt.check(t, cfg) + }) + } +} + func TestLoaderApplyOverlayLayersDefaultsValidatesAndRecordsProvenance(t *testing.T) { path := filepath.Join(t.TempDir(), "config.toml") if err := os.WriteFile(path, []byte("bot_user = \"widget\"\n[[repos]]\nowner = \"acme\"\nname = \"app\"\n"), 0o600); err != nil { From f1f7745001df8b88d4651dfeb7f228fdad54ef9c Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Sat, 19 Sep 2026 03:07:14 +1000 Subject: [PATCH 2/4] fix(config): clone the Services map so an overlay cannot mutate the published snapshot Config.Clone deep-copies every reference-type field except Services, which became a map[string]ServiceConnection. A value copy shares the map header, and yaml.Unmarshal writes keys into whatever map it is handed -- so Loader.ApplyOverlay, which clones precisely so it cannot corrupt the base document, rewrote the published snapshot in place: ApplyOverlay(doc, {"services": {"state": {"target": "10.0.0.5:9090"}}}) left doc.Config.Services["state"].Target changed That defeats the reason Clone exists: a failed or partial overlay must not alter the configuration still being served. Red: TestConfigCloneDeepCopiesReferenceFields failed with "Services map is shared" after the case was added; green with the one-line clone. --- internal/config/clone_test.go | 7 +++++++ internal/config/config.go | 5 +++++ 2 files changed, 12 insertions(+) diff --git a/internal/config/clone_test.go b/internal/config/clone_test.go index dd3f613db..8240b4995 100644 --- a/internal/config/clone_test.go +++ b/internal/config/clone_test.go @@ -32,6 +32,9 @@ func TestConfigCloneDeepCopiesReferenceFields(t *testing.T) { Hosted: map[string]ImageHostedProvider{"openai": {Enabled: true}}, Local: map[string]ImageLocalProvider{"sdxl": {Enabled: true}}, }, + Services: Services{ + ServiceNameState: {Target: "127.0.0.1:9090", TargetToken: "secret"}, + }, } got := orig.Clone() @@ -52,6 +55,7 @@ func TestConfigCloneDeepCopiesReferenceFields(t *testing.T) { *got.Tools.WebFetch.Enabled = false got.Image.Hosted["openai"] = ImageHostedProvider{Enabled: false} got.Image.Local["sdxl"] = ImageLocalProvider{Enabled: false} + got.Services[ServiceNameState] = ServiceConnection{Target: "changed"} if orig.Models["builder"] != "m" { t.Error("Models map is shared") @@ -95,4 +99,7 @@ func TestConfigCloneDeepCopiesReferenceFields(t *testing.T) { if !orig.Image.Local["sdxl"].Enabled { t.Error("Image.Local map is shared") } + if orig.Services[ServiceNameState].TargetToken != "secret" { + t.Error("Services map is shared") + } } diff --git a/internal/config/config.go b/internal/config/config.go index 7f169ecab..2627b3665 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -665,6 +665,11 @@ func (c Config) Clone() Config { c.Memory.ProviderConfig = maps.Clone(c.Memory.ProviderConfig) c.Image.Hosted = maps.Clone(c.Image.Hosted) c.Image.Local = maps.Clone(c.Image.Local) + // Services is a map of structs, so the header is shared by the value copy + // above and yaml.Unmarshal writes keys into whatever map it is handed. Without + // this line an overlay -- or a dashboard PATCH -- rewrites the PUBLISHED + // snapshot in place, which is the mutation Clone exists to prevent. + c.Services = maps.Clone(c.Services) if c.Tools.WebFetch.Enabled != nil { v := *c.Tools.WebFetch.Enabled c.Tools.WebFetch.Enabled = &v From 1284cb69f6673a8c5ad9ebbbd0b0cca9868ac681 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Fri, 18 Sep 2026 23:07:36 +1000 Subject: [PATCH 3/4] docs(docsgen): drop a duplicated line in the obsoleteArtifacts comment --- tools/docsgen/main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/docsgen/main.go b/tools/docsgen/main.go index adf639820..110adab8d 100644 --- a/tools/docsgen/main.go +++ b/tools/docsgen/main.go @@ -202,7 +202,6 @@ func compareArtifacts(relative string, expected, committed []byte) []string { return problems } -// obsoleteArtifacts reports committed generated files in the artifact's // obsoleteArtifacts reports files present in a directory docsgen owns that the // generator no longer produces, so a renamed or deleted artifact cannot linger // as dead documentation. A directory docsgen does not own is never judged: its From 73f019ea9f1d965ac76be87759c4463ebd5814f7 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Sat, 19 Sep 2026 03:07:38 +1000 Subject: [PATCH 4/4] docs: correct the withdrawn State Store topology and the authority revision MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two of the three findings from review. The third had already been fixed by the version of the service-registry PRD that landed on main. 1. docs/prds/state-store-contract.md §10 still prescribed the topology that was deleted when the State Store was extracted. It said an empty `Target` opens a local `*store.Store` adapter, and that the daemon's base path is that local store "already opened by openProductionTaskStore". Neither holds: the standalone archie-state-store process exclusively owns archie.db, and both archied and archie-gateway reject an empty target (bootstrap.go, state_store_client.go: "services.state.target is required"). openProductionTaskStore still exists, but only inside that standalone process. Following the ratified section would have reinstated the deleted in-process ownership path. The config.example.toml bullet (which still advised adding the section with a COMMENTED target -- impossible to boot) and the "opposite default (local, not remote)" blockquote are corrected with it; the section is already present with the target uncommented and marked REQUIRED. 2. CLAUDE.md named the State Store authority rev. 2c while the contract itself is ratified at rev. 2d. AGENTS.md is a symlink to CLAUDE.md, so one edit covers both guides. Not changed: the typo-visibility finding is already addressed by the PRD on main. Its tracking name, unregisteredServiceKeys, is unimplemented in Go, so the runtime diagnostic is still missing -- filed as archie-core-jp4g. --- CLAUDE.md | 2 +- docs/prds/state-store-contract.md | 36 +++++++++++++++++++------------ 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index ce7282cd1..cd2f3e350 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -183,7 +183,7 @@ structures found in legacy packages. `providerRegistry.RegisterOptional`. Missing or failed providers log warnings and degrade health without terminating the process. - **`internal/infrastructure/staterpc/` (State Store gRPC contract):** - Authority is `docs/prds/state-store-contract.md` (rev. 2c) -- read it before + Authority is `docs/prds/state-store-contract.md` (rev. 2d) -- read it before changing this package or its callers. - The proto (`proto/state/v1/state.proto`, service `StateStoreService`, package `statev1` in `internal/contracts/state/v1/`) is one gRPC service diff --git a/docs/prds/state-store-contract.md b/docs/prds/state-store-contract.md index f51cd95d3..a7f07e545 100644 --- a/docs/prds/state-store-contract.md +++ b/docs/prds/state-store-contract.md @@ -549,22 +549,30 @@ an explicit operator decision. registered once via `RegisterService(context, name, target, listen, tokenEnv)`. The TOML shape below is unchanged, so nothing else in this section is affected; read `cfg.Services.Get("state")` wherever it says `cfg.Services.State`. -- **Empty `Target` → local** `*store.Store` adapter (the default; State Store not yet - extracted). **Set `Target` → dial gRPC** and use `*staterpc.Client`. This differs from the - gateway, where `Target` is always defaulted to `127.0.0.1:8585` because the gateway is already - a separate process. -- Because `[services.state]` is **not** in `config.example.toml` today (the gateway's own - section is also missing — documentation debt), **add `[services.state]`** with a commented - `target` to `config.example.toml` in the same change that introduces the config field, so the - seam is visible to operators. -- Composition root: `internal/app/archied/bootstrap.go`. Base path = the local `*store.Store` - (already opened by `openProductionTaskStore`); if `cfg.Services.State.Target != ""`, dial gRPC - and assign `b.stateStore = staterpc.NewClient(conn)` (with the token from config/secret). +- **Empty `Target` → startup error** (corrected in rev. 2d). This bullet previously said an + empty `Target` opens a local `*store.Store` adapter. That default was withdrawn when the + State Store was extracted (§12 step 7): the standalone `archie-state-store` process + exclusively owns `archie.db`, so both `archied` and `archie-gateway` now reject an empty + target (`internal/app/archied/bootstrap.go` and `state_store_client.go`: `services.state.target + is required`). **Set `Target` → dial gRPC** with `*staterpc.Client`; no client opens the + database itself. This differs from the gateway, where `Target` is defaulted to + `127.0.0.1:8585` because `archie-gateway` is already a separate process whose client and + server share one default. +- `[services.state]` is present and **uncommented** in `config.example.toml`, with `target` + marked REQUIRED, because the value is now load-bearing for every client. (This bullet + previously instructed adding the section with a commented-out `target`; that advice is + withdrawn, since a commented target cannot boot.) +- Composition root: `internal/app/archied/bootstrap.go` (the daemon) and + `internal/app/archieui/config.go` (the UI process) both read `cfg.Services.Get("state")` and + dial the remote State Store. `openProductionTaskStore` is **not** called from the daemon: it + lives in the standalone `archie-state-store` process (`internal/app/archied/state_store.go`), + which is the only path that opens `archie.db`. > **Mode fork, decided.** The gateway uses a single `target` address (no enum). The State Store -> boundary uses the same seam for symmetry, but with the opposite default (local, not remote) -> because the State Store is not yet extracted. No `mode` field is added. A companion -> `STATE_STORE_TOKEN` secret (or a `[services.state] target_token` key) carries authentication. +> boundary uses the same seam for symmetry, and **both now require a non-empty `target`**: the +> State Store's local-store default was withdrawn when the process was extracted (§12 step 7). +> No `mode` field is added. A companion `STATE_STORE_TOKEN` secret (or a +> `[services.state] target_token` key) carries authentication. ---