fix(config): stop a partial overlay clearing the fields it omits

An overlay or dashboard PATCH naming only some fields of a map-valued config entry
silently zeroed the rest. For [services.<name>] that meant losing the bearer token,
so changing a State Store endpoint dropped its authentication:

  start   Services{"state": {Target: "127.0.0.1:50051", TargetToken: "secret"}}
  overlay {"services": {"state": {"target": "10.0.0.5:50051"}}}
  result  Services{"state": {Target: "10.0.0.5:50051", TargetToken: ""}}

Cause: ApplyOverlayValues did yaml.Marshal(overrides) then yaml.Unmarshal into cfg.
yaml gives a struct FIELD the field-level treatment the doc comment promises, but a
map VALUE is decoded into a fresh zero value, so an entry the overlay only partly
addresses loses every field it does not name. That contract was written when
Services was a struct; making it map[string]ServiceConnection changed the behaviour
underneath it.

Fix at the producer: fold the fields the overlay omits forward out of the entry cfg
already holds, so the decoder is handed a complete entry. Recursion follows the
destination type, so the promise now holds for every map of structs -- services,
image.hosted, image.local and providers -- in one place instead of at each field's
reader. Struct field-level merge, scalar-valued map-key merge and slice replacement
are unchanged, and overrides is read, never mutated.

The regression test also converts to the map form, which is what unbreaks the build:
apply_test.go still built Services with the struct form after services.go became a
map, and because the two live in different files the breakage merged cleanly. There
is no Go build or test CI, so nothing compiled it.

Red: four cases failed with the token, the target and the sibling fields zeroed.
Green after the fold.
This commit is contained in:
sam committed 2026-09-19 03:07:14 +10:00
1 parent 2385e81066
commit 369dfdf550
2 files changed
+231 -12

No files matched your search

+151 -1
View File
@@ -2,6 +2,9 @@ package configuration
import (
"fmt"
"maps"
"reflect"
"strings"
"gopkg.in/yaml.v3"
@@ -18,11 +21,26 @@ import (
// dashboard PATCH path calls it on a copy of the published config and
// validates the materialised result before persisting. Either way the
// caller owns the cfg value -- pass a copy when the base must survive.
//
// overrides is read, never mutated.
func ApplyOverlayValues(cfg *config.Config, overrides map[string]any) error {
if len(overrides) == 0 {
return nil
}
data, err := yaml.Marshal(overrides)
// yaml gives a struct field the field-level treatment promised above, but a
// map VALUE is decoded into a fresh zero value: an entry the overlay only
// partially addresses would lose every field it does not name, so an
// operator changing services.state.target alone would silently drop
// target_token. Fold the fields the overlay omits forward out of the entry
// cfg already holds, leaving the decode a complete entry to read. Doing it
// here, once, is what makes the promise hold for every map of structs
// (services, image.hosted, image.local, providers) instead of at each
// field's reader.
folded, err := foldOverrides(reflect.ValueOf(cfg), overrides)
if err != nil {
return err
}
data, err := yaml.Marshal(folded)
if err != nil {
return fmt.Errorf("%w: encoding config overlay: %w", ErrUnreadable, err)
}
@@ -31,3 +49,135 @@ func ApplyOverlayValues(cfg *config.Config, overrides map[string]any) error {
}
return nil
}
// foldOverrides returns doc with the fields its map entries omit carried
// forward from the entries cfg already holds. It descends through structs, so
// {"image": {"hosted": {...}}} folds the same way as an override written at the
// top level.
//
// Only mapping-valued map entries need it: yaml keeps the entries a map of
// scalars does not name, and a struct field it does not name keeps its value.
func foldOverrides(cfg reflect.Value, doc map[string]any) (map[string]any, error) {
if cfg.Kind() == reflect.Pointer {
if cfg.IsNil() {
return doc, nil
}
cfg = cfg.Elem()
}
if cfg.Kind() != reflect.Struct {
return doc, nil
}
out := make(map[string]any, len(doc))
maps.Copy(out, doc)
for key, value := range out {
field, ok := yamlField(cfg, key)
if !ok {
continue
}
sub, ok := value.(map[string]any)
if !ok {
continue // a scalar replaces the whole field, and the decode says so
}
switch field.Kind() {
case reflect.Struct:
folded, err := foldOverrides(field, sub)
if err != nil {
return nil, err
}
out[key] = folded
case reflect.Map:
folded, err := foldMapEntries(field, sub)
if err != nil {
return nil, err
}
out[key] = folded
}
}
return out, nil
}
// foldMapEntries returns one map field's override with each entry the overlay
// names carried forward from the entry of the same key in cfg.
func foldMapEntries(field reflect.Value, doc map[string]any) (map[string]any, error) {
if field.IsNil() || field.Type().Key().Kind() != reflect.String {
return doc, nil
}
out := make(map[string]any, len(doc))
maps.Copy(out, doc)
for key, value := range out {
entry, ok := value.(map[string]any)
if !ok {
continue
}
existing := field.MapIndex(reflect.ValueOf(key).Convert(field.Type().Key()))
if !existing.IsValid() {
continue // an entry the overlay introduces has no earlier fields
}
carried, err := asMapping(existing)
if err != nil {
return nil, err
}
if carried == nil {
continue // not a mapping, so the overlay replaces it wholesale
}
out[key] = mergeMapping(carried, entry)
}
return out, nil
}
// asMapping renders one config value as the mapping the decode will read it
// back as, which is what lets the fields an override omits be carried forward.
// It returns nil when the value does not encode to a mapping.
func asMapping(value reflect.Value) (map[string]any, error) {
data, err := yaml.Marshal(value.Interface())
if err != nil {
return nil, fmt.Errorf("%w: encoding config overlay: %w", ErrUnreadable, err)
}
// Decoding into any cannot fail for marshalled output; a value that is not
// a mapping simply yields no mapping.
var generic any
_ = yaml.Unmarshal(data, &generic)
mapping, _ := generic.(map[string]any)
return mapping, nil
}
// mergeMapping layers override over base: mappings merge key by key, anything
// else replaces, matching how the decode treats a struct field versus a scalar.
// It allocates rather than mutating either argument.
func mergeMapping(base, override map[string]any) map[string]any {
out := make(map[string]any, len(base)+len(override))
maps.Copy(out, base)
for key, value := range override {
if over, ok := value.(map[string]any); ok {
if under, ok := base[key].(map[string]any); ok {
out[key] = mergeMapping(under, over)
continue
}
}
out[key] = value
}
return out
}
// yamlField returns the field of struct v that the decode reads key into. The
// yaml tag is the one consulted because the decode below is yaml's.
func yamlField(v reflect.Value, key string) (reflect.Value, bool) {
t := v.Type()
for i := range t.NumField() {
field := t.Field(i)
if field.PkgPath != "" {
continue // unexported, so the decoder cannot write it
}
name, _, _ := strings.Cut(field.Tag.Get("yaml"), ",")
if name == "-" {
continue
}
if name == "" {
name = strings.ToLower(field.Name)
}
if name == key {
return v.Field(i), true
}
}
return reflect.Value{}, false
}
@@ -2,6 +2,7 @@ package configuration
import (
"log/slog"
"maps"
"os"
"path/filepath"
"testing"
@@ -38,7 +39,7 @@ func TestApplyOverlayValuesFieldLevelMerge(t *testing.T) {
// must not clear target_token, because dropping the token silently removes
// authentication for a non-loopback State Store. The existing field-level test
// only covers one level (budgets.max_steps); these cases go two levels deep into
// a struct of structs, which is the shape Services has.
// a map of structs, which is the shape Services has.
func TestApplyOverlayValuesPreservesOmittedNestedFields(t *testing.T) {
tests := []struct {
name string
@@ -48,21 +49,27 @@ func TestApplyOverlayValuesPreservesOmittedNestedFields(t *testing.T) {
}{
{
name: "target-only overlay keeps the token",
start: config.Services{State: config.ServiceConnection{Target: "127.0.0.1:50051", TargetToken: "secret"}},
overlay: map[string]any{"services": map[string]any{"state": map[string]any{"target": "10.0.0.5:50051"}}},
want: config.Services{State: config.ServiceConnection{Target: "10.0.0.5:50051", TargetToken: "secret"}},
start: config.Services{config.ServiceNameState: {Target: "127.0.0.1:50051", TargetToken: "secret"}},
overlay: map[string]any{"services": map[string]any{config.ServiceNameState: map[string]any{"target": "10.0.0.5:50051"}}},
want: config.Services{config.ServiceNameState: {Target: "10.0.0.5:50051", TargetToken: "secret"}},
},
{
name: "token-only overlay keeps the target",
start: config.Services{State: config.ServiceConnection{Target: "10.0.0.5:50051", TargetToken: "old"}},
overlay: map[string]any{"services": map[string]any{"state": map[string]any{"target_token": "rotated"}}},
want: config.Services{State: config.ServiceConnection{Target: "10.0.0.5:50051", TargetToken: "rotated"}},
start: config.Services{config.ServiceNameState: {Target: "10.0.0.5:50051", TargetToken: "old"}},
overlay: map[string]any{"services": map[string]any{config.ServiceNameState: map[string]any{"target_token": "rotated"}}},
want: config.Services{config.ServiceNameState: {Target: "10.0.0.5:50051", TargetToken: "rotated"}},
},
{
name: "sibling service is untouched",
start: config.Services{Gateway: config.ServiceConnection{Target: "127.0.0.1:50052"}, State: config.ServiceConnection{Target: "a", TargetToken: "t"}},
overlay: map[string]any{"services": map[string]any{"state": map[string]any{"target": "b"}}},
want: config.Services{Gateway: config.ServiceConnection{Target: "127.0.0.1:50052"}, State: config.ServiceConnection{Target: "b", TargetToken: "t"}},
start: config.Services{config.ServiceNameGateway: {Target: "127.0.0.1:50052"}, config.ServiceNameState: {Target: "a", TargetToken: "t"}},
overlay: map[string]any{"services": map[string]any{config.ServiceNameState: map[string]any{"target": "b"}}},
want: config.Services{config.ServiceNameGateway: {Target: "127.0.0.1:50052"}, config.ServiceNameState: {Target: "b", TargetToken: "t"}},
},
{
name: "an entry the overlay introduces decodes in full",
start: config.Services{},
overlay: map[string]any{"services": map[string]any{config.ServiceNameState: map[string]any{"target": "127.0.0.1:50051", "target_token": "fresh"}}},
want: config.Services{config.ServiceNameState: {Target: "127.0.0.1:50051", TargetToken: "fresh"}},
},
}
@@ -72,13 +79,75 @@ func TestApplyOverlayValuesPreservesOmittedNestedFields(t *testing.T) {
if err := ApplyOverlayValues(&cfg, tt.overlay); err != nil {
t.Fatalf("ApplyOverlayValues: %v", err)
}
if cfg.Services != tt.want {
if !maps.Equal(cfg.Services, tt.want) {
t.Errorf("Services = %+v, want %+v", cfg.Services, tt.want)
}
})
}
}
// TestApplyOverlayValuesMergesEveryMapEntry pins that the preservation above is
// a property of ApplyOverlayValues rather than of Services. yaml.v3 decodes a
// map value into a fresh zero value, so every map[string]struct field loses the
// fields an override omits; Services, image.hosted and providers are the three
// the configuration has, so a fix that named one of them would leave the other
// two silently zeroing operator settings.
func TestApplyOverlayValuesMergesEveryMapEntry(t *testing.T) {
tests := []struct {
name string
start config.Config
overlay map[string]any
check func(t *testing.T, cfg config.Config)
}{
{
name: "image.hosted entry keeps the fields the overlay omits",
start: config.Config{Image: config.ImageConfig{Hosted: map[string]config.ImageHostedProvider{
"minimax": {Enabled: true, Class: "minimax", APIKeyEnv: "MINIMAX_API_KEY"},
}}},
overlay: map[string]any{"image": map[string]any{"hosted": map[string]any{
"minimax": map[string]any{"base_url": "https://api.example"},
}}},
check: func(t *testing.T, cfg config.Config) {
t.Helper()
want := map[string]config.ImageHostedProvider{
"minimax": {Enabled: true, Class: "minimax", APIKeyEnv: "MINIMAX_API_KEY", BaseURL: "https://api.example"},
}
if !maps.Equal(cfg.Image.Hosted, want) {
t.Errorf("Image.Hosted = %+v, want %+v", cfg.Image.Hosted, want)
}
},
},
{
name: "providers entry keeps the fields the overlay omits",
start: config.Config{Providers: map[string]config.Provider{
"anthropic": {Class: "anthropic", APIKeyEnv: "ANTHROPIC_API_KEY"},
}},
overlay: map[string]any{"providers": map[string]any{
"anthropic": map[string]any{"base_url": "https://proxy.example"},
}},
check: func(t *testing.T, cfg config.Config) {
t.Helper()
want := map[string]config.Provider{
"anthropic": {Class: "anthropic", APIKeyEnv: "ANTHROPIC_API_KEY", BaseURL: "https://proxy.example"},
}
if !maps.Equal(cfg.Providers, want) {
t.Errorf("Providers = %+v, want %+v", cfg.Providers, want)
}
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := tt.start
if err := ApplyOverlayValues(&cfg, tt.overlay); err != nil {
t.Fatalf("ApplyOverlayValues: %v", err)
}
tt.check(t, cfg)
})
}
}
func TestLoaderApplyOverlayLayersDefaultsValidatesAndRecordsProvenance(t *testing.T) {
path := filepath.Join(t.TempDir(), "config.toml")
if err := os.WriteFile(path, []byte("bot_user = \"widget\"\n[[repos]]\nowner = \"acme\"\nname = \"app\"\n"), 0o600); err != nil {