fix(config): keep the fields a file overlay omits, and judge a database-owned setting at its owner

Resolves archie-core-xrux and archie-core-i3qm. The -config-overlay file path
(and the overlay-directory path) now fold a partial map-valued entry instead of
replacing it wholesale, so an overlay that sets one sibling key no longer clears
services.state.target_token. The file load path stops judging settings the
control plane owns, so a stale TOML value in a database-owned field can no
longer stop archie-state-store starting; boot.runtimeConfig remains the one
layering call and still validates the merged document. A kind with no stored
value leaves the file's value in force.
This commit is contained in:
sam committed 2026-09-21 22:15:22 +10:00
commit d5c9c8016d
17 files changed
+1398 -139

No files matched your search

+9 -1
View File
@@ -708,9 +708,17 @@ func seedStoreResources(t *testing.T, st *store.Store, configPath string) {
if err != nil {
t.Fatalf("build control plane server: %v", err)
}
if _, err := server.ImportConfig(t.Context(), doc.Config); err != nil {
_, skipped, err := server.ImportConfig(t.Context(), doc.Config)
if err != nil {
t.Fatalf("seed store resources: %v", err)
}
if len(skipped) > 0 {
// This helper stands in for the State Store's own seeding, which seeds
// every kind it can and leaves the rest absent. A skipped kind here
// would mean the fixture store is missing a resource it claims to hold,
// and the verdict under test would be about a different store.
t.Fatalf("seed store resources: %d kind(s) refused: %+v", len(skipped), skipped)
}
}
// storedResourceCount reads the count validate reports out of its summary line.
@@ -8,6 +8,8 @@ import (
"time"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"github.com/samcharles93/archie-core/internal/app/controlplane"
"github.com/samcharles93/archie-core/internal/config"
@@ -18,7 +20,9 @@ import (
// controlPlaneStub answers Query from a fixed resource map. queryErr, when
// set, fails every Query so a test can drive the unreachable-State-Store
// path.
// path. A kind the map does not carry is answered the way the server answers
// one with no stored value -- codes.NotFound (controlplane.mapError) -- so a
// test can drive the absent-resource state a skipped seed leaves behind.
type controlPlaneStub struct {
values map[string]any
queryErr error
@@ -32,8 +36,12 @@ func (c *controlPlaneStub) Query(_ context.Context, request *pb.QueryRequest, _
if c.queryErr != nil {
return nil, c.queryErr
}
value, err := json.Marshal(c.values[request.Kind])
return &pb.QueryResponse{Resource: &pb.Resource{Kind: request.Kind, Version: 3, ValueJson: value}}, err
value, ok := c.values[request.Kind]
if !ok {
return nil, status.Error(codes.NotFound, "resource not found")
}
encoded, err := json.Marshal(value)
return &pb.QueryResponse{Resource: &pb.Resource{Kind: request.Kind, Version: 3, ValueJson: encoded}}, err
}
func (*controlPlaneStub) History(context.Context, *pb.HistoryRequest, ...grpc.CallOption) (*pb.HistoryResponse, error) {
@@ -67,11 +75,15 @@ func databaseOwnedResources() map[string]any {
// fileConfig is what the loader produces from config.toml alone: the layer a
// SIGHUP reload re-resolves, holding none of the database's values. It is a
// config the daemon would run, so it passes configuration.Validate -- the live
// apply path runs that same check over the snapshot it is about to publish.
// apply path runs that same check over the snapshot it is about to publish --
// and every field validation reads is filled in the way the loader's defaults
// fill it, so a kind with no stored value can fall back to this document (see
// TestRuntimeConfigKeepsTheFileValueWhenAKindHasNoStoredResource).
func fileConfig() config.Config {
return config.Config{
BotUser: "widget",
Forge: config.Forge{Type: "github"},
Dispatch: config.Dispatch{Trigger: "assignee"},
Providers: map[string]config.Provider{"file": {Class: "openai"}},
Models: map[string]string{"builder": "file/model"},
Repos: []config.Repo{{Owner: "acme", Name: "from-file"}},
@@ -81,7 +93,6 @@ func fileConfig() config.Config {
PluginDir: "/file/plugins",
Containers: config.ContainerConfig{Image: "archie:from-file", PullPolicy: "missing"},
Budgets: config.Budgets{MaxSteps: 1, WallClock: config.Duration(time.Minute)},
Dispatch: config.Dispatch{Trigger: "assignee"},
}
}
@@ -0,0 +1,253 @@
package archied
import (
"context"
"errors"
"fmt"
"log/slog"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/samcharles93/archie-core/internal/app/controlplane"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
)
// staleDatabaseOwnedSettings are TOML fragments, each carrying one value a
// control-plane resource is seeded from and replaces wholesale. They are the
// values docs/prds/runtime-control-plane.md,
// "Bootstrap, migration, and recovery" says must not block a process: "after
// migration, settings in TOML are ignored and cannot block State Store
// startup".
var staleDatabaseOwnedSettings = []struct {
name string
body string
}{
{
name: "provider base_url carrying userinfo",
body: "[providers.openai]\nclass = \"openai\"\nbase_url = \"https://token@example.com/v1\"\n",
},
{
name: "negative poll_interval",
body: "poll_interval = \"-5s\"\n",
},
{
name: "unrecognised dispatch.trigger",
body: "[dispatch]\ntrigger = \"labels\"\n",
},
{
name: "negative containers.volume_ttl",
body: "[containers]\nvolume_ttl = \"-1m\"\n",
},
{
name: "malformed repos test_glob",
body: "[[repos]]\nowner = \"acme\"\nname = \"app\"\ntest_glob = \"[\"\n",
},
}
// TestLoadConfigAcceptsStaleDatabaseOwnedValues is the first of the two layers
// archie-state-store boots through. loadConfig resolves the file and is where
// the loader used to reject these values; it must not, because the setting
// belongs to the control plane. The second layer is the boot itself, covered by
// TestStateStoreBootsWithStaleDatabaseOwnedValues -- passing only this one is
// not the item's claim.
func TestLoadConfigAcceptsStaleDatabaseOwnedValues(t *testing.T) {
for _, tt := range staleDatabaseOwnedSettings {
t.Run(tt.name, func(t *testing.T) {
path := t.TempDir() + "/config.toml"
writeConfig(t, path, "bot_user = \"widget\"\n"+tt.body)
b := &boot{log: slog.New(slog.DiscardHandler)}
if err := b.loadConfig(t.Context(), path, ""); err != nil {
t.Fatalf("loadConfig: %v (a stale database-owned value must not fail boot)", err)
}
})
}
}
// staleBootGrace is how long one boot in
// TestStateStoreBootsWithStaleDatabaseOwnedValues may run before the test cancels
// it: short, because the assertion does not depend on it -- a refusal arrives as
// an error either before the cancel (first select) or after it (second select,
// where a non-Canceled error fails the test).
//
// staleShutdownGrace bounds the shutdown, and is deliberately not the 5s the
// single-boot regression test uses: this table runs five boots back to back on a
// host that may be busy, and the select returns the moment the process does, so
// the bound costs nothing when shutdown is quick and only decides how long a
// failure takes to report.
const (
staleBootGrace = 250 * time.Millisecond
staleShutdownGrace = 30 * time.Second
)
// TestStateStoreBootsWithStaleDatabaseOwnedValues is the process-level half of
// the item, and the layer that actually decides it: archie-state-store seeds
// every control-plane resource from the same file config on a fresh database
// (state_store.go, control.ImportConfig), and that seed is what runs the
// resource's own validator. A seed that does not validate is skipped and
// reported, not fatal -- the kind stays absent, the file's value is the one in
// effect, and the operator's fix in config.toml is what clears it.
func TestStateStoreBootsWithStaleDatabaseOwnedValues(t *testing.T) {
for _, tt := range staleDatabaseOwnedSettings {
t.Run(tt.name, func(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "config.toml")
body := fmt.Sprintf(
"bot_user = 'archie'\ndb_path = %q\n%s[forge]\ntype = 'github'\nhost = 'https://github.example.com'\n",
filepath.Join(dir, "state.db"), tt.body,
)
if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
t.Fatal(err)
}
// Loopback, so no token is required. Cancellation is the assertion:
// a boot that refuses the config returns before it, and one that
// serves returns only when the context ends. Cancel first, then
// assert -- racing startup against a wall clock is a coin flip.
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
errCh := make(chan error, 1)
go func() {
errCh <- RunStateStore(ctx, StateStoreOptions{Config: path, Listen: "127.0.0.1:0"})
}()
select {
case err := <-errCh:
t.Fatalf("RunStateStore returned before cancellation, want it to serve: %v", err)
case <-time.After(staleBootGrace):
}
cancel()
select {
case err := <-errCh:
if err != nil && !errors.Is(err, context.Canceled) {
t.Fatalf("RunStateStore returned %v, want nil or context.Canceled", err)
}
case <-time.After(staleShutdownGrace):
t.Fatal("RunStateStore did not return within the shutdown grace after cancellation")
}
})
}
}
// TestRuntimeConfigKeepsTheFileValueWhenAKindHasNoStoredResource is the other
// end of a seed the control plane refused: the kind stays ABSENT rather than
// being stored (ImportConfig skips it), and boot must then fall back to the
// file document's value. Treating an absent kind as fatal is how a config the
// file layer blesses left the daemon unbootable with a database-named error,
// and left editing config.toml unable to fix it.
func TestRuntimeConfigKeepsTheFileValueWhenAKindHasNoStoredResource(t *testing.T) {
tests := []struct {
name string
kind string
check func(t *testing.T, cfg config.Config)
}{
{
name: "repository policies",
kind: controlplane.RepositoryPoliciesKind,
check: func(t *testing.T, cfg config.Config) {
t.Helper()
if len(cfg.Repos) != 1 || cfg.Repos[0].Name != "from-file" {
t.Errorf("Repos = %+v, want the file's from-file", cfg.Repos)
}
},
},
{
name: "scheduling policy",
kind: controlplane.SchedulingPolicyKind,
check: func(t *testing.T, cfg config.Config) {
t.Helper()
if cfg.PollInterval != config.Duration(time.Minute) {
t.Errorf("PollInterval = %v, want the file's 1m", cfg.PollInterval)
}
},
},
{
name: "container runtime policies",
kind: controlplane.ContainerRuntimePoliciesKind,
check: func(t *testing.T, cfg config.Config) {
t.Helper()
if cfg.Containers.Image != "archie:from-file" {
t.Errorf("Containers.Image = %q, want the file's archie:from-file", cfg.Containers.Image)
}
},
},
{
name: "providers",
kind: controlplane.ProviderSettingsKind,
check: func(t *testing.T, cfg config.Config) {
t.Helper()
if got := cfg.Providers["file"].Class; got != "openai" {
t.Errorf("Providers[file].Class = %q, want the file's openai", got)
}
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
resources := databaseOwnedResources()
delete(resources, tt.kind)
b := newReloadBoot(t, &controlPlaneStub{values: resources})
if err := b.loadRuntimeConfig(t.Context()); err != nil {
t.Fatalf("loadRuntimeConfig: %v (a kind with no stored value must leave the file's value in effect)", err)
}
tt.check(t, b.cfgHolder.Get())
})
}
}
// TestDuplicateRepositoriesFailOnTheFilesOwnMessage is the end of the path the
// fix-closure review found. A duplicate [[repos]] entry is refused by the
// repository-policies validator, so the kind is never seeded and stays absent --
// and boot must then fail on the FILE's copy of the value, naming the duplicate,
// with the file as the fix. It used to fail on "control-plane resource not
// found": a database-named error that editing config.toml could not clear.
func TestDuplicateRepositoriesFailOnTheFilesOwnMessage(t *testing.T) {
resources := databaseOwnedResources()
delete(resources, controlplane.RepositoryPoliciesKind)
b := newReloadBoot(t, &controlPlaneStub{values: resources})
b.cfg.Repos = []config.Repo{{Owner: "acme", Name: "app"}, {Owner: "acme", Name: "app"}}
err := b.loadRuntimeConfig(t.Context())
if err == nil {
t.Fatal("loadRuntimeConfig accepted a duplicate repository list")
}
if !errors.Is(err, configuration.ErrInvalidInput) {
t.Errorf("loadRuntimeConfig = %v, want it to wrap configuration.ErrInvalidInput", err)
}
if !strings.Contains(err.Error(), "duplicates") {
t.Errorf("loadRuntimeConfig = %v, want the duplicate named the way configuration.Validate names it", err)
}
if strings.Contains(err.Error(), "not found") {
t.Errorf("loadRuntimeConfig = %v, want no absent-resource error: the file's value is the one in effect", err)
}
}
// TestRuntimeConfigFailsClosedOnAnInvalidDatabaseValue is the other half of the
// same change: the checks that no longer run at load run on the effective
// document, at the one place database resources are layered in. A stored value
// that will not validate must stop archied starting rather than be published as
// the running configuration -- a state nothing can recover from in band.
func TestRuntimeConfigFailsClosedOnAnInvalidDatabaseValue(t *testing.T) {
resources := databaseOwnedResources()
resources[controlplane.ProviderSettingsKind] = map[string]any{
"main": map[string]any{"class": "openai", "base_url": "https://token@example.com/v1"},
}
b := newReloadBoot(t, &controlPlaneStub{values: resources})
err := b.loadRuntimeConfig(t.Context())
if err == nil {
t.Fatal("loadRuntimeConfig accepted a database value that does not validate")
}
if !errors.Is(err, configuration.ErrInvalidInput) {
t.Errorf("loadRuntimeConfig = %v, want it to wrap configuration.ErrInvalidInput", err)
}
if got := b.cfgHolder.Get().Providers["main"].BaseURL; got == "https://token@example.com/v1" {
t.Error("the invalid database value was published as the running configuration")
}
}
+5 -1
View File
@@ -30,8 +30,12 @@ func minimalConfigTOML(botUser string) string {
return "bot_user = \"" + botUser + "\"\n[[repos]]\nowner = \"acme\"\nname = \"app\"\n"
}
// invalidConfigTOML carries a value the bootstrap document still owns, so
// resolving it fails. The settings the control plane owns are no longer judged
// at load, so using one of those here would silently turn these tests into a
// false pass.
func invalidConfigTOML() string {
return "bot_user = \"widget\"\ndispatch = { trigger = \"bogus-trigger\" }\n[[repos]]\nowner = \"acme\"\nname = \"app\"\n"
return "bot_user = \"widget\"\n[memory]\nengine = \"not-a-real-engine\"\n[[repos]]\nowner = \"acme\"\nname = \"app\"\n"
}
func TestReloadAppliesAndPublishes(t *testing.T) {
+25 -1
View File
@@ -106,10 +106,11 @@ func RunStateStore(ctx context.Context, options StateStoreOptions) error {
if err != nil {
return err
}
versions, err := control.ImportConfig(ctx, b.cfg)
versions, skipped, err := control.ImportConfig(ctx, b.cfg)
if err != nil {
return fmt.Errorf("import control-plane resources: %w", err)
}
b.reportUnseededResources(skipped)
if err := migrateLegacySchedules(ctx, control, b.cfg.DBPath, versions[controlplane.SchedulesKind]); err != nil {
return fmt.Errorf("migrate schedules: %w", err)
}
@@ -142,6 +143,29 @@ func RunStateStore(ctx context.Context, options StateStoreOptions) error {
return serveStateStore(ctx, listener, deps, opts)
}
// reportUnseededResources logs every kind ImportConfig could not seed.
//
// Reported, not fatal: a stale TOML value in a setting the database owns must
// not stop this process from starting (docs/prds/runtime-control-plane.md,
// "Bootstrap, migration, and recovery"). The kind stays ABSENT rather than
// half-written, so the file document's value is the one in effect for every
// reader -- the layering leaves a kind with no stored value alone
// (controlplane.resourceReader) -- and the operator's fix is still the file.
// Whether the value may RUN is decided where it is used: boot.runtimeConfig
// validates the effective document and refuses to start archied or the Gateway
// with one it cannot run. The seed is retried on this process's next start, so
// a corrected config.toml takes effect when archie-state-store restarts.
//
// It is a method rather than a loop in RunStateStore because the boot sequence
// is at its complexity budget: the branch belongs to the report's own policy,
// not to the sequence that triggers it.
func (b *boot) reportUnseededResources(skipped []controlplane.SeedSkip) {
for _, skip := range skipped {
b.log.Error("control-plane resource not seeded; the kind stays absent and the file's value is the one in effect",
"kind", skip.Kind, "err", skip.Err)
}
}
// openStateStoreControlPlane builds the State Store's control plane server: the
// validating side of the workflow step vocabulary, registered here at the
// composition root before the first definition is read or replaced. The same
+54 -7
View File
@@ -3,6 +3,7 @@ package controlplane
import (
"context"
"encoding/json"
"errors"
"testing"
"time"
@@ -74,7 +75,7 @@ func TestImportConfigSeedsWorkflowDefinitionsWithoutOverwritingOverride(t *testi
resources := store.OpenTest(t)
defer resources.Close()
server := testServer(t, resources)
if _, err := server.ImportConfig(t.Context(), config.Config{}); err != nil {
if _, _, err := server.ImportConfig(t.Context(), config.Config{}); err != nil {
t.Fatal(err)
}
resource, err := resources.Resource(t.Context(), WorkflowDefinitionsKind)
@@ -96,7 +97,7 @@ func TestImportConfigSeedsWorkflowDefinitionsWithoutOverwritingOverride(t *testi
if _, err := resources.PutResource(t.Context(), store.ResourceWrite{Kind: WorkflowDefinitionsKind, Value: value, ExpectedVersion: resource.Version, Actor: "test", Source: "test", RequestID: "override"}); err != nil {
t.Fatal(err)
}
if _, err := server.ImportConfig(t.Context(), config.Config{}); err != nil {
if _, _, err := server.ImportConfig(t.Context(), config.Config{}); err != nil {
t.Fatal(err)
}
got, err := resources.Resource(t.Context(), WorkflowDefinitionsKind)
@@ -114,7 +115,7 @@ func TestImportConfigSeedsPersonasWithoutOverwritingEdits(t *testing.T) {
resources := store.OpenTest(t)
defer resources.Close()
server := testServer(t, resources)
if _, err := server.ImportConfig(t.Context(), config.Config{}); err != nil {
if _, _, err := server.ImportConfig(t.Context(), config.Config{}); err != nil {
t.Fatal(err)
}
resource, err := resources.Resource(t.Context(), PersonasKind)
@@ -133,7 +134,7 @@ func TestImportConfigSeedsPersonasWithoutOverwritingEdits(t *testing.T) {
if _, err := resources.PutResource(t.Context(), store.ResourceWrite{Kind: PersonasKind, Value: value, ExpectedVersion: resource.Version, Actor: "test", Source: "test", RequestID: "persona-edit"}); err != nil {
t.Fatal(err)
}
if _, err := server.ImportConfig(t.Context(), config.Config{}); err != nil {
if _, _, err := server.ImportConfig(t.Context(), config.Config{}); err != nil {
t.Fatal(err)
}
got, err := resources.Resource(t.Context(), PersonasKind)
@@ -178,9 +179,15 @@ func TestRegistryRoutesAndValidatesDefinitions(t *testing.T) {
resources := store.OpenTest(t)
defer resources.Close()
server := testServer(t, resources)
if _, err := server.ImportConfig(t.Context(), config.Config{Containers: config.ContainerConfig{PullPolicy: "missing"}}); err != nil {
// The image is required, here as in the file document: a seed without one is
// skipped rather than stored, and the Command below would then fail on the
// missing resource instead of on the pull policy it exists to reject.
if _, _, err := server.ImportConfig(t.Context(), config.Config{Containers: config.ContainerConfig{Image: "archie-agent:test", PullPolicy: "missing"}}); err != nil {
t.Fatal(err)
}
if _, err := resources.Resource(t.Context(), ContainerRuntimePoliciesKind); err != nil {
t.Fatalf("container runtime policies were not seeded: %v", err)
}
catalog, err := server.Catalog(t.Context(), &pb.CatalogRequest{})
if err != nil {
@@ -209,7 +216,7 @@ func TestProviderSeedKeepsReferencesAndNeverResolvedSecrets(t *testing.T) {
defer resources.Close()
server := testServer(t, resources)
cfg := config.Config{Providers: map[string]config.Provider{"openai": {Class: "openai", APIKeyEnv: "OPENAI_API_KEY", APIKey: config.SecretRef{Engine: "env", Key: "OPENAI_API_KEY"}}}}
if _, err := server.ImportConfig(context.Background(), cfg); err != nil {
if _, _, err := server.ImportConfig(context.Background(), cfg); err != nil {
t.Fatal(err)
}
resource, err := resources.Resource(t.Context(), ProviderSettingsKind)
@@ -235,7 +242,7 @@ func TestHistoryCarriesEveryRevisionWithItsAudit(t *testing.T) {
resources := store.OpenTest(t)
defer resources.Close()
server := testServer(t, resources)
if _, err := server.ImportConfig(t.Context(), config.Config{}); err != nil {
if _, _, err := server.ImportConfig(t.Context(), config.Config{}); err != nil {
t.Fatal(err)
}
seeded, err := server.Query(t.Context(), &pb.QueryRequest{Kind: WorkflowExecutionSettingsKind})
@@ -269,3 +276,43 @@ func TestHistoryCarriesEveryRevisionWithItsAudit(t *testing.T) {
t.Fatalf("history for an unknown kind = %v, want NotFound", err)
}
}
// TestImportConfigSkipsASeedItCannotValidate is the reason a bad seed is not
// fatal. docs/prds/runtime-control-plane.md, "Bootstrap, migration, and
// recovery": after migration, settings in TOML are ignored and cannot block
// State Store startup -- and ImportConfig is on that startup path. The value is
// not stored either, so the setting stays file-owned: the file's value is still
// the one in effect, and editing config.toml is still the fix. Writing the
// invalid value instead would hand ownership to the database and leave the
// operator with a file edit that does nothing.
func TestImportConfigSkipsASeedItCannotValidate(t *testing.T) {
t.Parallel()
resources := store.OpenTest(t)
defer resources.Close()
server := testServer(t, resources)
// Every other field is filled in the way the loader fills it (Validate
// documents that it does not apply defaults), so the malformed glob is the
// only seed that can be refused.
cfg := validConfigForValidation()
cfg.Repos = []config.Repo{{Owner: "acme", Name: "app", TestGlob: "["}}
versions, skipped, err := server.ImportConfig(t.Context(), cfg)
if err != nil {
t.Fatalf("ImportConfig: %v, want a skipped seed rather than a fatal one", err)
}
if _, stored := versions[RepositoryPoliciesKind]; stored {
t.Error("the resource was seeded, want the invalid seed skipped")
}
if len(skipped) != 1 || skipped[0].Kind != RepositoryPoliciesKind || skipped[0].Err == nil {
t.Fatalf("skipped = %+v, want the repository policies and the reason", skipped)
}
if _, err := resources.Resource(t.Context(), RepositoryPoliciesKind); !errors.Is(err, store.ErrResourceNotFound) {
t.Errorf("repository policies Resource = %v, want ErrResourceNotFound: an invalid seed must not be stored", err)
}
// Kinds that are fine are still seeded: one skipped kind does not abandon
// the rest of the migration.
if _, err := resources.Resource(t.Context(), WorkflowDefinitionsKind); err != nil {
t.Errorf("workflow definitions: %v, want them seeded alongside the skipped kind", err)
}
}
@@ -6,6 +6,7 @@ import (
"time"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
)
const (
@@ -40,25 +41,27 @@ func operationalDefinitions() []Definition {
{Kind: ToolSettingsKind, Title: "Tool and MCP settings", ApplyMode: "restart-required", Schema: objectSchema, Seed: seedTools, Validate: validateTools},
{Kind: PluginSettingsKind, Title: "Plugin settings", ApplyMode: "restart-required", Schema: objectSchema, Seed: func(cfg config.Config) any {
return pluginSettings{cfg.PluginDir, cfg.ModuleDir, cfg.SecretEngineDir, cfg.SkillsDir}
}, Validate: func(input []byte) error { return validateAs(input, func(pluginSettings) error { return nil }) }},
}, Validate: validatePluginSettings},
{Kind: ContainerRuntimePoliciesKind, Title: "Container runtime policies", ApplyMode: "restart-required", Schema: objectSchema, Seed: func(cfg config.Config) any { return cfg.Containers }, Validate: validateContainers},
}
}
// validatePluginSettings accepts every value: the four directories are free-form
// operator paths with no cross-field rule and no rule the configuration package
// applies either (configuration.Validate has no say over them), so there is
// nothing yet for this validator to enforce. It stays a real function rather than
// an inline no-op so the place to add a rule is obvious when one exists -- e.g.
// "a path that is absolute" or "a skills dir under the plugin dir".
func validatePluginSettings(input []byte) error {
return validateAs(input, func(pluginSettings) error { return nil })
}
func validateRepositories(input []byte) error {
return validateAs(input, func(repos []config.Repo) error {
seen := make(map[string]struct{}, len(repos))
for _, repo := range repos {
if strings.TrimSpace(repo.Owner) == "" || strings.TrimSpace(repo.Name) == "" {
return fmt.Errorf("repository owner and name are required")
}
if _, exists := seen[repo.FullName()]; exists {
return fmt.Errorf("duplicate repository %q", repo.FullName())
}
seen[repo.FullName()] = struct{}{}
}
return nil
})
// The rules live in the configuration package and are shared with the file
// document's own validation (configuration.ValidateRepositories): which
// repository lists are valid cannot differ between the file that seeds this
// resource and the resource that replaces it.
return validateAs(input, configuration.ValidateRepositories)
}
func validateScheduling(input []byte) error {
@@ -66,11 +69,18 @@ func validateScheduling(input []byte) error {
if policy.MaxRetries < 0 {
return fmt.Errorf("max_retries must not be negative")
}
if policy.PollInterval != "0s" {
interval, err := time.ParseDuration(policy.PollInterval)
if err != nil || interval < 0 {
return fmt.Errorf("poll_interval must be a non-negative duration")
}
// A positive interval, not a non-negative one: the file layer's rule is
// the same, and there is no defaulting behind a stored value -- a stored
// "0s" reaches cfg.PollInterval as zero and stops archied starting.
interval, err := time.ParseDuration(policy.PollInterval)
if err != nil || interval <= 0 {
return fmt.Errorf("poll_interval must be a positive duration")
}
// Same reason as the repository rules above: RuntimeConfig replaces
// cfg.Dispatch from this resource, and the daemon refuses to start with a
// trigger it does not know.
if !configuration.DispatchTriggerValid(policy.Dispatch.Trigger) {
return fmt.Errorf("dispatch.trigger %q is not a discovery rule the daemon can poll with", policy.Dispatch.Trigger)
}
return nil
})
@@ -84,6 +94,11 @@ func validateContainers(input []byte) error {
if settings.PullPolicy != "" && settings.PullPolicy != "missing" && settings.PullPolicy != "always" {
return fmt.Errorf("pull_policy must be missing or always")
}
// The image is required for autonomous workflow workers; the same rule
// holds for the stored policies that replace the file's [containers].
if strings.TrimSpace(settings.Image) == "" {
return fmt.Errorf("container image is required for autonomous workflow workers")
}
return nil
})
}
+35 -5
View File
@@ -61,8 +61,30 @@ func (d Definition) Decode(input []byte) ([]byte, error) {
return encoded, nil
}
func (s *Server) ImportConfig(ctx context.Context, cfg config.Config) (map[string]int64, error) {
// SeedSkip records a resource ImportConfig could not seed from the file config,
// and why. It is reported rather than returned as an error: see ImportConfig.
type SeedSkip struct {
Kind string
Err error
}
// ImportConfig seeds every control-plane resource that has no stored value yet
// from the file config, and reports the version of every kind it left alone.
//
// A seed the resource's own validator refuses is SKIPPED and returned in skipped,
// not fatal. Nothing is written for that kind, so it stays ABSENT and the file
// document's value stays the one in effect (Client.RuntimeConfig leaves an absent
// kind alone), with the fix still in the file. The seed is retried on the next
// start of this process, so correcting config.toml re-seeds it.
// docs/prds/runtime-control-plane.md, "Bootstrap, migration, and recovery":
// after migration, settings in TOML are ignored and cannot block State Store
// startup -- and this import runs on the State Store's startup path. Fail closed
// belongs to the process that uses the value: boot.runtimeConfig validates the
// effective document and refuses to start archied with a value it cannot run
// with, which is what keeps an invalid value from taking effect silently.
func (s *Server) ImportConfig(ctx context.Context, cfg config.Config) (map[string]int64, []SeedSkip, error) {
versions := make(map[string]int64, len(s.ordered))
var skipped []SeedSkip
for _, definition := range s.ordered {
resource, err := s.store.Resource(ctx, definition.Kind)
if err == nil {
@@ -70,19 +92,27 @@ func (s *Server) ImportConfig(ctx context.Context, cfg config.Config) (map[strin
continue
}
if !errors.Is(err, store.ErrResourceNotFound) {
return nil, err
return nil, nil, err
}
value, err := definition.seededValue(cfg)
if err != nil {
return nil, err
// Skipped and reported, not fatal: nothing is written, so the kind
// stays absent and the file document's value is the one in effect,
// while the process that would RUN the value still fails closed on
// it (boot.runtimeConfig validates the effective document).
// seededValue wraps a seed's encode and validation failures together,
// which is why both take this path: the alternative is a second seed
// implementation here to tell them apart.
skipped = append(skipped, SeedSkip{Kind: definition.Kind, Err: err})
continue
}
resource, err = s.store.PutResource(ctx, store.ResourceWrite{Kind: definition.Kind, Value: value, Actor: "system:migration", Source: "legacy-config", RequestID: "import:" + definition.Kind, ExpectedVersion: 0, At: time.Now().UTC()})
if err != nil {
return nil, fmt.Errorf("seed %s: %w", definition.Kind, err)
return nil, nil, fmt.Errorf("seed %s: %w", definition.Kind, err)
}
versions[definition.Kind] = resource.Version
}
return versions, nil
return versions, skipped, nil
}
// seededValue is the value the State Store writes for a kind it does not hold
+53 -23
View File
@@ -13,12 +13,21 @@ import (
)
// resourceReader is the one read the layering performs: a resource kind's value
// and the version it came from. The gRPC client and the in-process server both
// provide it, so "what the database covers in the running config" has exactly
// one implementation. The offline validate runs the same code as boot, which is
// the only way its verdict can be boot's verdict.
// and the version it came from, or nothing at all when the store holds no value
// for that kind. The gRPC client and the in-process server both provide it, so
// "what the database covers in the running config" has exactly one
// implementation. The offline validate runs the same code as boot, which is the
// only way its verdict can be boot's verdict.
//
// found is false only for the reader that dials a live State Store: a kind with
// no stored value leaves the file document's value in effect instead of failing
// the process, which is the state a seed ImportConfig refused leaves behind
// (controlplane.Server.ImportConfig skips it) and the state of a database the
// migration has not reached yet. storeReader never reports absent -- it answers
// with the seed the State Store would write -- so the offline verdict stays
// boot's.
type resourceReader interface {
query(ctx context.Context, kind string, decode func([]byte) error) (int64, error)
query(ctx context.Context, kind string, decode func([]byte) error) (int64, bool, error)
}
// RuntimeConfig applies restart-scoped database resources over bootstrap
@@ -67,27 +76,28 @@ type storeReader struct {
seeds map[string][]byte
}
func (r storeReader) query(ctx context.Context, kind string, decode func([]byte) error) (int64, error) {
func (r storeReader) query(ctx context.Context, kind string, decode func([]byte) error) (int64, bool, error) {
resource, err := r.resources.Resource(ctx, kind)
if errors.Is(err, store.ErrResourceNotFound) {
seed, ok := r.seeds[kind]
if !ok {
return 0, fmt.Errorf("read %s: %w", kind, store.ErrResourceNotFound)
return 0, false, fmt.Errorf("read %s: %w", kind, store.ErrResourceNotFound)
}
if err := decode(seed); err != nil {
return 0, fmt.Errorf("decode %s seed: %w", kind, err)
return 0, false, fmt.Errorf("decode %s seed: %w", kind, err)
}
// No version: nothing has been stored for this kind, so nothing has
// been applied either.
return 0, nil
// been applied either. It is still a value to layer in -- the seed --
// which is why this reader never reports the kind absent.
return 0, true, nil
}
if err != nil {
return 0, fmt.Errorf("read %s: %w", kind, err)
return 0, false, fmt.Errorf("read %s: %w", kind, err)
}
if err := decode(resource.Value); err != nil {
return 0, fmt.Errorf("decode %s: %w", kind, err)
return 0, false, fmt.Errorf("decode %s: %w", kind, err)
}
return resource.Version, nil
return resource.Version, true, nil
}
func runtimeConfigFrom(ctx context.Context, reader resourceReader, base config.Config) (config.Config, map[string]int64, error) {
@@ -171,7 +181,7 @@ func runtimeToolConfigFrom(ctx context.Context, reader resourceReader, versions
func runtimeChatConfigFrom(ctx context.Context, reader resourceReader, base config.ChatConfig) (config.ChatConfig, int64, error) {
out := base
version, err := reader.query(ctx, ChannelSettingsKind, func(value []byte) error {
version, found, err := reader.query(ctx, ChannelSettingsKind, func(value []byte) error {
var settings channelSettings
if err := json.Unmarshal(value, &settings); err != nil {
return err
@@ -186,17 +196,31 @@ func runtimeChatConfigFrom(ctx context.Context, reader resourceReader, base conf
}
return nil
})
return out, version, err
if err != nil {
return out, 0, err
}
if !found {
return out, 0, nil
}
return out, version, nil
}
// layerResource decodes a resource and records the version it came from, so the
// layering ends up holding the version of every kind it applied.
//
// A kind with no stored value is not an error and records no version: the file
// document's value stays in effect. That is the state a seed the resource
// validator refused leaves behind (controlplane.Server.ImportConfig skips it),
// and failing here instead would stop the process with a database-named error
// that editing config.toml cannot clear.
func layerResource(ctx context.Context, reader resourceReader, versions map[string]int64, kind string, decode func([]byte) error) error {
version, err := reader.query(ctx, kind, decode)
version, found, err := reader.query(ctx, kind, decode)
if err != nil {
return err
}
versions[kind] = version
if found {
versions[kind] = version
}
return nil
}
@@ -205,17 +229,23 @@ func layerResourceJSON(ctx context.Context, reader resourceReader, versions map[
}
// query returns the version of the resource it decoded, so callers that layer
// a resource in can report the version they applied.
func (c *Client) query(ctx context.Context, kind string, decode func([]byte) error) (int64, error) {
// a resource in can report the version they applied. A kind the store holds no
// value for is reported as not found rather than as an error: the caller leaves
// the file document's value in effect (see resourceReader).
func (c *Client) query(ctx context.Context, kind string, decode func([]byte) error) (int64, bool, error) {
response, err := c.rpc.Query(ctx, &pb.QueryRequest{Kind: kind})
if err != nil {
return 0, clientError(err)
mapped := clientError(err)
if errors.Is(mapped, ErrNotFound) {
return 0, false, nil
}
return 0, false, mapped
}
if response.Resource == nil {
return 0, fmt.Errorf("%s resource missing", kind)
return 0, false, fmt.Errorf("%s resource missing", kind)
}
if err := decode(response.Resource.ValueJson); err != nil {
return 0, fmt.Errorf("decode %s: %w", kind, err)
return 0, false, fmt.Errorf("decode %s: %w", kind, err)
}
return response.Resource.Version, nil
return response.Resource.Version, true, nil
}
@@ -0,0 +1,173 @@
package controlplane
import (
"encoding/json"
"testing"
"time"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
)
// TestResourceValidatorsRejectWhatEffectiveValidationRejects ties the two rule
// sets that guard the same fields. A control-plane resource is seeded from the
// file document and then replaces it wholesale (Client.RuntimeConfig), so a
// value the file path accepts must not be storable through the resource, and a
// value the resource accepts must not stop the daemon's validation of the
// effective document. Guards that live only on one side are how a stale TOML
// value becomes a database value the operator cannot fix by editing config.toml.
//
// Each case mutates the same field the resource carries, so the two halves
// cannot drift apart without failing here.
func TestResourceValidatorsRejectWhatEffectiveValidationRejects(t *testing.T) {
tests := []struct {
name string
kind string
value any
mutate func(cfg *config.Config)
wantErr bool
}{
{
name: "valid scheduling policy",
kind: SchedulingPolicyKind,
value: map[string]any{"poll_interval": "1m0s", "max_retries": 0, "dispatch": map[string]any{"trigger": "assignee"}},
mutate: func(cfg *config.Config) {},
wantErr: false,
},
{
name: "unrecognised dispatch trigger",
kind: SchedulingPolicyKind,
value: map[string]any{"poll_interval": "1m0s", "max_retries": 0, "dispatch": map[string]any{"trigger": "labels"}},
mutate: func(cfg *config.Config) { cfg.Dispatch.Trigger = "labels" },
wantErr: true,
},
{
name: "negative poll interval",
kind: SchedulingPolicyKind,
value: map[string]any{"poll_interval": "-5s", "max_retries": 0, "dispatch": map[string]any{"trigger": "assignee"}},
mutate: func(cfg *config.Config) { cfg.PollInterval = config.Duration(-5 * time.Second) },
wantErr: true,
},
{
// A zero interval is what applyDefaults replaces before validation
// runs, so the file layer only ever sees it as a value to reject. The
// resource has no defaulting behind it: a stored "0s" would become
// cfg.PollInterval = 0 and stop archied starting.
name: "zero poll interval",
kind: SchedulingPolicyKind,
value: map[string]any{"poll_interval": "0s", "max_retries": 0, "dispatch": map[string]any{"trigger": "assignee"}},
mutate: func(cfg *config.Config) { cfg.PollInterval = 0 },
wantErr: true,
},
{
name: "valid repository policies",
kind: RepositoryPoliciesKind,
value: []config.Repo{{Owner: "acme", Name: "app", TestGlob: "**/*_test.go"}},
mutate: func(cfg *config.Config) {},
wantErr: false,
},
{
name: "malformed test glob",
kind: RepositoryPoliciesKind,
value: []config.Repo{{Owner: "acme", Name: "app", TestGlob: "["}},
mutate: func(cfg *config.Config) { cfg.Repos = []config.Repo{{Owner: "acme", Name: "app", TestGlob: "["}} },
wantErr: true,
},
{
// Both layers have to call a duplicate list invalid. When only the
// resource did, a config the file layer blessed left the daemon
// unbootable on a database-named error.
name: "duplicate repository",
kind: RepositoryPoliciesKind,
value: []config.Repo{{Owner: "acme", Name: "app"}, {Owner: "acme", Name: "app"}},
mutate: func(cfg *config.Config) {
cfg.Repos = []config.Repo{{Owner: "acme", Name: "app"}, {Owner: "acme", Name: "app"}}
},
wantErr: true,
},
{
name: "valid container runtime policies",
kind: ContainerRuntimePoliciesKind,
value: config.ContainerConfig{Image: "archie-agent:test", PullPolicy: "missing"},
mutate: func(cfg *config.Config) {},
wantErr: false,
},
{
name: "container runtime policies without an image",
kind: ContainerRuntimePoliciesKind,
value: config.ContainerConfig{PullPolicy: "missing"},
mutate: func(cfg *config.Config) { cfg.Containers.Image = "" },
wantErr: true,
},
{
name: "negative container volume ttl",
kind: ContainerRuntimePoliciesKind,
value: config.ContainerConfig{Image: "archie-agent:test", VolumeTTL: config.Duration(-time.Minute)},
mutate: func(cfg *config.Config) { cfg.Containers.VolumeTTL = config.Duration(-time.Minute) },
wantErr: true,
},
{
name: "provider base URL carrying userinfo",
kind: ProviderSettingsKind,
value: map[string]any{"acme": map[string]any{"class": "openai", "base_url": "https://user:pass@example.com"}},
mutate: func(cfg *config.Config) {
cfg.Providers = map[string]config.Provider{"acme": {Class: "openai", BaseURL: "https://user:pass@example.com"}}
},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := validConfigForValidation()
tt.mutate(&cfg)
configErr := configuration.Validate(&cfg)
encoded, err := json.Marshal(tt.value)
if err != nil {
t.Fatal(err)
}
resourceErr := decodeResource(t, tt.kind, encoded)
if !tt.wantErr {
if configErr != nil {
t.Errorf("configuration.Validate = %v, want nil for a valid value", configErr)
}
if resourceErr != nil {
t.Errorf("%s Decode = %v, want nil for a valid value", tt.kind, resourceErr)
}
return
}
if configErr == nil {
t.Errorf("configuration.Validate accepted the value, want the effective document to reject it")
}
if resourceErr == nil {
t.Errorf("%s accepted the value, want the resource validator to reject what the effective document rejects", tt.kind)
}
})
}
}
// decodeResource runs the resource validator exactly as ImportConfig does when
// it seeds a kind from the file config (registry.go), against the production
// server the composition roots build (testServer).
func decodeResource(t *testing.T, kind string, encoded []byte) error {
t.Helper()
_, err := testServer(t, nil).definitions[kind].Decode(encoded)
return err
}
// validConfigForValidation is a document every check passes, so each case above
// fails only for the value it changes -- same fixture the configuration package
// uses, restated here because Validate documents that it does not apply
// defaults.
func validConfigForValidation() config.Config {
return config.Config{
BotUser: "archie-bot",
PollInterval: config.Duration(60 * time.Second),
Dispatch: config.Dispatch{Trigger: "assignee"},
Forge: config.Forge{Type: "none"},
Containers: config.ContainerConfig{Image: "ghcr.io/samcharles93/archie-agent:latest"},
Services: config.Services{config.ServiceNameGateway: {Target: "127.0.0.1:8585"}},
}
}
+64 -16
View File
@@ -27,16 +27,46 @@ func ApplyOverlayValues(cfg *config.Config, overrides map[string]any) error {
if len(overrides) == 0 {
return nil
}
// yaml gives a struct field the field-level treatment promised above, but a
// map VALUE is decoded into a fresh zero value: an entry the overlay only
// partially addresses would lose every field it does not name, so an
// operator changing services.state.target alone would silently drop
// target_token. Fold the fields the overlay omits forward out of the entry
// cfg already holds, leaving the decode a complete entry to read. Doing it
// here, once, is what makes the promise hold for every map of structs
// (services, image.hosted, image.local, providers) instead of at each
// field's reader.
folded, err := foldOverrides(reflect.ValueOf(cfg), overrides)
return applyOverlayMapping(cfg, overrides)
}
// applyOverlayFile layers the overlay file at path over target, with the same
// field-level precedence ApplyOverlayValues gives a map overlay, and reports the
// file's keys that nothing consumes. target is whatever the overlay addresses:
// the whole config for a main file, one sub-struct for a feature file.
//
// Both processed overlay forms go through it -- the single overlay file, and an
// overlay directory's main and feature files -- so neither can lose the fields of
// a map-valued entry it only partly addresses. A conf.d/*.yaml extra in an
// overlay directory does not: decodeExtra stores a whole file under
// cfg.Extra[name] and nothing reads that map, so there is no entry to fold into.
// Decoding an overlay into target directly replaces such an entry wholesale,
// which is how an overlay naming only services.state.target cleared the
// target_token the base file set.
func applyOverlayFile(path string, target any) ([]string, error) {
mapping, err := decodeFileMapping(path)
if err != nil {
return nil, err
}
if err := applyOverlayMapping(target, mapping); err != nil {
return nil, err
}
return overlayFileKeys(path, mapping, structTypeOf(target))
}
// applyOverlayMapping is the decode both overlay paths share: the fields each
// map-valued entry the overlay does not name are folded forward out of the
// entry target already holds, and the completed mapping is then decoded over it.
//
// yaml gives a struct field the field-level treatment ApplyOverlayValues
// promises, but a map VALUE is decoded into a fresh zero value: an entry the
// overlay only partially addresses would lose every field it does not name, so
// an operator changing services.state.target alone would silently drop
// target_token. Folding here, once, is what makes the promise hold for every
// map of structs (services, image.hosted, image.local, providers) instead of
// at each field's reader.
func applyOverlayMapping(target any, doc map[string]any) error {
folded, err := foldOverrides(reflect.ValueOf(target), doc)
if err != nil {
return err
}
@@ -44,12 +74,22 @@ func ApplyOverlayValues(cfg *config.Config, overrides map[string]any) error {
if err != nil {
return fmt.Errorf("%w: encoding config overlay: %w", ErrUnreadable, err)
}
if err := yaml.Unmarshal(data, cfg); err != nil {
if err := yaml.Unmarshal(data, target); err != nil {
return fmt.Errorf("%w: parsing config overlay: %w", ErrUnreadable, err)
}
return nil
}
// structTypeOf returns the struct type target points at, so the fold and the key
// report can inspect it without an instance.
func structTypeOf(target any) reflect.Type {
t := reflect.TypeOf(target)
for t != nil && t.Kind() == reflect.Pointer {
t = t.Elem()
}
return t
}
// foldOverrides returns doc with the fields its map entries omit carried
// forward from the entries cfg already holds. It descends through structs, so
// {"image": {"hosted": {...}}} folds the same way as an override written at the
@@ -162,9 +202,17 @@ func mergeMapping(base, override map[string]any) map[string]any {
// yamlField returns the field of struct v that the decode reads key into. The
// yaml tag is the one consulted because the decode below is yaml's.
func yamlField(v reflect.Value, key string) (reflect.Value, bool) {
t := v.Type()
for i := range t.NumField() {
field := t.Field(i)
field, ok := yamlFieldOf(v.Type(), key)
if !ok {
return reflect.Value{}, false
}
return v.FieldByIndex(field.Index), true
}
// yamlFieldOf is yamlField without a value: the overlay key report walks a raw
// mapping against the target's type before anything is decoded into it.
func yamlFieldOf(t reflect.Type, key string) (reflect.StructField, bool) {
for field := range t.Fields() {
if field.PkgPath != "" {
continue // unexported, so the decoder cannot write it
}
@@ -176,8 +224,8 @@ func yamlField(v reflect.Value, key string) (reflect.Value, bool) {
name = strings.ToLower(field.Name)
}
if name == key {
return v.Field(i), true
return field, true
}
}
return reflect.Value{}, false
return reflect.StructField{}, false
}
@@ -27,6 +27,32 @@ func decodeConfigFileKeys(path string, target any) ([]string, error) {
}
}
// decodeFileMapping parses a configuration file into the nested mapping it
// carries, with no typed target. An overlay needs this shape: a typed decode
// cannot distinguish a key the file omits from a key it sets to the zero
// value, and that distinction is what carrying the fields of a partially
// addressed map entry forward is built on (see applyOverlayFile).
func decodeFileMapping(path string) (map[string]any, error) {
var doc map[string]any
switch filepath.Ext(path) {
case ".yaml", ".yml":
data, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("%w: reading %s: %w", ErrUnreadable, path, err)
}
if err := yaml.Unmarshal(data, &doc); err != nil {
return nil, fmt.Errorf("%w: parsing %s: %w", ErrUnreadable, path, err)
}
case ".toml":
if _, err := toml.DecodeFile(path, &doc); err != nil {
return nil, fmt.Errorf("%w: parsing %s: %w", ErrUnreadable, path, err)
}
default:
return nil, fmt.Errorf("%w: config file %s must end in .toml, .yaml, or .yml", ErrUnreadable, path)
}
return doc, nil
}
// decodeTOMLKeys decodes a TOML file into target, additionally
// returning the top-level dotted key paths present in the file that
// target did not consume (toml.MetaData.Undecoded()). A single decode
@@ -72,15 +98,25 @@ func decodeYAML(path string, target any) error {
// into cfg directly. Memory and tools are sub-structs whose keys sit at the
// top level of their own file rather than nested under a "memory:" or
// "tools:" key, so they decode into the sub-struct.
func decodeFeature(cfg *config.Config, feature Feature, path string) error {
//
// An overlay layer's feature file is folded over the base layer, for the same
// reason and with the same precedence as a main overlay file. The keys that fold
// reports are dropped here: only a main file reports unknown keys today
// (decodeMain), and a feature file reporting a second, differently-shaped set
// would be a new surface rather than a fix to this one.
func decodeFeature(cfg *config.Config, feature Feature, path string, layer Layer) error {
target := any(cfg)
switch feature {
case FeatureMemory:
return decodeYAML(path, &cfg.Memory)
target = &cfg.Memory
case FeatureTools:
return decodeYAML(path, &cfg.Tools)
default:
return decodeYAML(path, cfg)
target = &cfg.Tools
}
if layer == LayerOverlay {
_, err := applyOverlayFile(path, target)
return err
}
return decodeYAML(path, target)
}
// decodeExtra decodes an unrecognised conf.d/ file into cfg.Extra under name.
+100 -12
View File
@@ -5,6 +5,7 @@ import (
"log/slog"
"os"
"path/filepath"
"reflect"
"sort"
"github.com/samcharles93/archie-core/internal/config"
@@ -146,7 +147,7 @@ func (l *Loader) ApplyOverlay(doc *Document, overrides map[string]any) (*Documen
next.Config = doc.Config.Clone()
next.Provenance.Origins = append([]Origin(nil), doc.Provenance.Origins...)
if len(overrides) == 0 {
return l.finalize(&next)
return l.finalize(&next, Validate)
}
if err := ApplyOverlayValues(&next.Config, overrides); err != nil {
return nil, err
@@ -155,7 +156,7 @@ func (l *Loader) ApplyOverlay(doc *Document, overrides map[string]any) (*Documen
return nil, err
}
next.Provenance.record(Origin{Path: "config_overlay (runtime)", Role: RoleMain, Layer: LayerOverlay})
return l.finalize(&next)
return l.finalize(&next, Validate)
}
func (l *Loader) overlayFile(basePath, overlayPath string) (*Document, error) {
@@ -173,7 +174,10 @@ func (l *Loader) overlayFile(basePath, overlayPath string) (*Document, error) {
doc.Provenance.record(Origin{Path: basePath, Role: RoleMain, Layer: LayerBase})
if overlayPath != "" {
cfgKeys, err = decodeConfigFileKeys(overlayPath, &doc.Config)
// The overlay is folded over the base config rather than decoded into
// it: a decode replaces a map-valued entry wholesale, clearing the
// fields of that entry the overlay does not name (applyOverlayFile).
cfgKeys, err = applyOverlayFile(overlayPath, &doc.Config)
if err != nil {
return nil, err
}
@@ -185,7 +189,72 @@ func (l *Loader) overlayFile(basePath, overlayPath string) (*Document, error) {
doc.Provenance.record(Origin{Path: overlayPath, Role: RoleMain, Layer: LayerOverlay})
}
return l.finalize(doc)
return l.finalize(doc, validateBootstrap)
}
// overlayFileKeys reports the keys of an overlay file that nothing consumes. It
// has two sources, because the file's own decode and the fold's apply decode do
// not match keys the same way: TOML accepts a key that differs only in case,
// while the yaml decode the fold uses does not. A key only the file's decode
// matches would therefore be consumed by the report below and dropped in
// silence by the apply, so the apply's own view of the mapping is unioned in.
//
// The second decode here is deliberate: it is what carries TOML's Undecoded()
// view of the file, which the raw mapping cannot report (it consumes every key).
func overlayFileKeys(path string, mapping map[string]any, target reflect.Type) ([]string, error) {
decoded, err := decodeConfigFileKeys(path, reflect.New(target).Interface())
if err != nil {
return nil, err
}
return append(decoded, unmatchableKeys(mapping, target)...), nil
}
// unmatchableKeys returns the dotted paths of the keys in mapping that the fold's
// yaml decode cannot match to a field of target. It descends the way foldOverrides
// does -- through structs and through string-keyed maps of structs -- so it sees
// the same keys the apply does. A slice of tables is not descended: the yaml
// decode replaces it wholesale, so anything inside it is a value, not a key.
func unmatchableKeys(mapping map[string]any, target reflect.Type) []string {
var out []string
for key, value := range mapping {
field, ok := yamlFieldOf(target, key)
if !ok {
out = append(out, key)
continue
}
sub, ok := value.(map[string]any)
if !ok {
continue
}
switch field.Type.Kind() {
case reflect.Struct:
out = append(out, prefixed(key, unmatchableKeys(sub, field.Type))...)
case reflect.Map:
entry := field.Type.Elem()
for entry.Kind() == reflect.Pointer {
entry = entry.Elem()
}
if entry.Kind() != reflect.Struct || field.Type.Key().Kind() != reflect.String {
continue
}
for entryName, entryValue := range sub {
entryMapping, ok := entryValue.(map[string]any)
if !ok {
continue
}
out = append(out, prefixed(key+"."+entryName, unmatchableKeys(entryMapping, entry))...)
}
}
}
return out
}
// prefixed qualifies a nested key report with the path that reached it.
func prefixed(prefix string, keys []string) []string {
for i, key := range keys {
keys[i] = prefix + "." + key
}
return keys
}
// unknownKeys returns the keys present in both a and b: one config file
@@ -275,7 +344,7 @@ func (l *Loader) Dir(baseDir, overlayDir string) (*Document, error) {
}
}
return l.finalize(doc)
return l.finalize(doc, validateBootstrap)
}
// loadDir discovers and decodes one directory into doc. The base layer must
@@ -288,7 +357,7 @@ func (l *Loader) loadDir(doc *Document, dir string, layer Layer) error {
switch path, isYAMLFile, ok := files.main(); {
case ok:
if err := l.decodeMain(doc, path, isYAMLFile); err != nil {
if err := l.decodeMain(doc, path, isYAMLFile, layer); err != nil {
return err
}
doc.Provenance.record(Origin{Path: path, Role: RoleMain, Layer: layer})
@@ -298,7 +367,7 @@ func (l *Loader) loadDir(doc *Document, dir string, layer Layer) error {
for _, feature := range files.sortedFeatures() {
path := files.features[feature]
if err := decodeFeature(&doc.Config, feature, path); err != nil {
if err := decodeFeature(&doc.Config, feature, path, layer); err != nil {
return err
}
doc.Provenance.record(Origin{Path: path, Role: RoleFeature, Layer: layer, Feature: feature})
@@ -315,8 +384,20 @@ func (l *Loader) loadDir(doc *Document, dir string, layer Layer) error {
return nil
}
// decodeMain decodes the daemon-level file in whichever format it uses.
func (l *Loader) decodeMain(doc *Document, path string, isYAMLFile bool) error {
// decodeMain decodes the daemon-level file in whichever format it uses. An
// overlay layer's file is FOLDED over what the base layer decoded, the same way
// a single overlay file folds over its base -- otherwise a map-valued entry it
// only partly addresses would clear the fields it does not name, which is the
// archie-core-e2e2 failure on the directory form of the same flag.
func (l *Loader) decodeMain(doc *Document, path string, isYAMLFile bool, layer Layer) error {
if layer == LayerOverlay {
cfgKeys, err := applyOverlayFile(path, &doc.Config)
if err != nil {
return err
}
doc.UnknownKeys = append(doc.UnknownKeys, cfgKeys...)
return decodeSchedulingFile(path, &doc.Scheduling)
}
if isYAMLFile {
if err := decodeYAML(path, &doc.Config); err != nil {
return err
@@ -336,12 +417,19 @@ func (l *Loader) decodeMain(doc *Document, path string, isYAMLFile bool) error {
}
// finalize applies defaults, then validates. The order matters: validation
// judges the effective configuration, including values the operator never
// judges the defaulted configuration, including values the operator never
// wrote.
func (l *Loader) finalize(doc *Document) (*Document, error) {
//
// validate is a parameter because the two kinds of caller produce different
// documents. Every file source ([Loader.File], [Loader.Overlay], [Loader.Dir])
// produces a BOOTSTRAP document and passes validateBootstrap: the settings the
// control plane owns are layered over it later, so a stale value in one of them
// must not fail the load. [Loader.ApplyOverlay] layers overrides onto a
// document that is already effective, so it passes [Validate].
func (l *Loader) finalize(doc *Document, validate func(*config.Config) error) (*Document, error) {
doc.UnknownKeys = sortedUnique(append(doc.UnknownKeys, unregisteredServiceKeys(doc.Config.Services)...))
l.applyDefaults(&doc.Config)
if err := Validate(&doc.Config); err != nil {
if err := validate(&doc.Config); err != nil {
return nil, err
}
l.log.Debug("configuration loaded", "sources", doc.Provenance.String())
@@ -236,31 +236,80 @@ func TestLoadTelegramTokenSecretRefAndLegacyFallback(t *testing.T) {
}
}
// TestLoadRejectsInvalidConfigEnumsAndGlobs pins which layer rejects each kind
// of invalid value. Every file source produces a BOOTSTRAP document, and the
// settings the control plane owns are layered over it later; a stale TOML value
// in one of those must not fail the load (docs/prds/runtime-control-plane.md,
// "Bootstrap, migration, and recovery"). So a check over a database-owned
// setting is asserted with Validate instead -- what boot.runtimeConfig runs over
// the document a process will use, and the checks the loader no longer applies.
// Each case asserts both halves it applies to, so a check that quietly
// disappeared fails the case rather than passing it.
func TestLoadRejectsInvalidConfigEnumsAndGlobs(t *testing.T) {
tests := []struct {
name string
extra string
name string
body string
wantLoadErr bool
}{
{name: "dispatch trigger", extra: "\n[dispatch]\ntrigger = \"labels\"\n"},
{name: "provider userinfo", extra: "\n[providers.openai]\nclass = \"openai\"\nbase_url = \"https://token@example.com/v1\"\n"},
{name: "provider query secret", extra: "\n[providers.openai]\nclass = \"openai\"\nbase_url = \"https://example.com/v1?api_key=secret\"\n"},
{name: "test glob", extra: "\n[[repos]]\nowner = \"acme\"\nname = \"app\"\ntest_glob = \"[\"\n"},
{
name: "dispatch trigger",
body: fileConfigPrefix + "[dispatch]\ntrigger = \"labels\"\n",
},
{
name: "provider userinfo",
body: fileConfigPrefix + "[providers.openai]\nclass = \"openai\"\nbase_url = \"https://token@example.com/v1\"\n",
},
{
name: "provider query secret",
body: fileConfigPrefix + "[providers.openai]\nclass = \"openai\"\nbase_url = \"https://example.com/v1?api_key=secret\"\n",
},
{
name: "test glob",
body: fileConfigPrefix + "[[repos]]\nowner = \"acme\"\nname = \"app\"\ntest_glob = \"[\"\n",
},
{
// File-owned settings: no control-plane resource carries them, so
// the load path is the layer that has to reject them.
name: "forge intake",
body: fileConfigPrefix + "[forge]\nintake = \"not-a-real-intake\"\n",
wantLoadErr: true,
},
{
name: "memory engine",
body: fileConfigPrefix + "[memory]\nengine = \"not-a-real-engine\"\n",
wantLoadErr: true,
},
{
name: "negative capture retention",
body: fileConfigPrefix + "[capture]\nretention = \"-1h\"\n",
wantLoadErr: true,
},
{
name: "image default naming no provider",
body: fileConfigPrefix + "[image]\ndefault = \"not-a-real-provider\"\n",
wantLoadErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
path := filepath.Join(t.TempDir(), "config.toml")
contents := "bot_user = \"widget\"\n"
if tt.name != "test glob" {
contents += "\n[[repos]]\nowner = \"acme\"\nname = \"app\"\n"
}
contents += tt.extra
if err := os.WriteFile(path, []byte(contents), 0o600); err != nil {
if err := os.WriteFile(path, []byte(tt.body), 0o600); err != nil {
t.Fatal(err)
}
if _, err := loadFile(path); err == nil {
t.Fatal("loadFile() succeeded, want validation error")
doc, err := New(nil).File(path)
if tt.wantLoadErr {
if err == nil {
t.Fatal("File() = nil error, want the bootstrap document rejected")
}
return
}
if err != nil {
t.Fatalf("File: %v (a database-owned value must not fail the load)", err)
}
if err := Validate(&doc.Config); err == nil {
t.Fatal("Validate = nil, want the value rejected there")
}
})
}
@@ -487,6 +536,11 @@ func TestLoadContainerVolumeTTL(t *testing.T) {
}
}
// TestLoadRejectsNegativeContainerVolumeTTL: containers are a control-plane
// resource (container-runtime-policies), so a negative volume_ttl is judged by
// Validate rather than by the load path -- a stale TOML value in a database-owned
// setting must not fail a process's startup. The second half of the assertion is
// what keeps dropping the check from passing this test.
func TestLoadRejectsNegativeContainerVolumeTTL(t *testing.T) {
path := filepath.Join(t.TempDir(), "config.toml")
contents := "bot_user = \"widget\"\n" +
@@ -497,8 +551,12 @@ func TestLoadRejectsNegativeContainerVolumeTTL(t *testing.T) {
t.Fatal(err)
}
if _, err := loadFile(path); err == nil {
t.Fatal("loadFile() accepted a negative containers.volume_ttl")
doc, err := New(nil).File(path)
if err != nil {
t.Fatalf("File: %v (a database-owned value must not fail the load)", err)
}
if err := Validate(&doc.Config); err == nil {
t.Fatal("Validate accepted a negative containers.volume_ttl")
}
}
@@ -571,6 +629,274 @@ func TestOverlay(t *testing.T) {
}
}
// TestResolveFileOverlayPreservesOmittedMapEntryFields characterizes the
// -config-overlay path at the depth the runtime overlay is already guarded at
// (TestApplyOverlayValuesPreservesOmittedNestedFields).
//
// It goes through Loader.Resolve, the production entry point, because the
// hazard was a property of how the overlay file is applied, not of any one
// decode helper: Loader.overlayFile decoded the overlay into the very config
// the base file produced, and a map-valued entry is replaced wholesale, so every
// field of that entry the overlay does not name was cleared.
//
// That is archie-core-e2e2 on a user-visible path. deployments/dev.toml is the
// documented -config-overlay argument and sets [services.state] target/listen,
// while the installed config.toml is where [services.state].target_token lives:
// the overlay silently dropped the token. The directory form of the same flag is
// covered by TestDirOverlayPreservesOmittedMapEntryFields.
func TestResolveFileOverlayPreservesOmittedMapEntryFields(t *testing.T) {
tests := []struct {
name string
base string
overlayName string
overlay string
check func(t *testing.T, cfg config.Config)
}{
{
// The deployment shape quoted above, with the value that is read
// only when a client dials a non-loopback State Store.
name: "services.state entry keeps the token it does not name",
base: "[services.state]\ntarget = \"127.0.0.1:9090\"\ntarget_token = \"secret\"\n",
overlay: "[services.state]\ntarget = \"10.0.0.5:9090\"\n",
check: func(t *testing.T, cfg config.Config) {
t.Helper()
got := cfg.Services[config.ServiceNameState]
if got.Target != "10.0.0.5:9090" {
t.Errorf("services.state.target = %q, want the overlay's 10.0.0.5:9090", got.Target)
}
if got.TargetToken != "secret" {
t.Errorf("services.state.target_token = %q, want the base's secret: a file overlay must not clear a field it does not name", got.TargetToken)
}
},
},
{
// An enabled hosted provider must keep its class and key env: the
// overlay naming only base_url would otherwise clear them and fail
// validation after the load.
name: "image.hosted entry keeps the fields it does not name",
base: "[image.hosted.minimax]\nenabled = true\nclass = \"minimax\"\n" +
"api_key_env = \"MINIMAX_API_KEY\"\n",
overlay: "[image.hosted.minimax]\nbase_url = \"https://api.example\"\n",
check: func(t *testing.T, cfg config.Config) {
t.Helper()
got := cfg.Image.Hosted["minimax"]
want := config.ImageHostedProvider{
Enabled: true,
Class: "minimax",
APIKeyEnv: "MINIMAX_API_KEY",
BaseURL: "https://api.example",
}
if got != want {
t.Errorf("image.hosted.minimax = %+v, want %+v", got, want)
}
},
},
{
name: "providers entry keeps the fields it does not name",
base: "[providers.anthropic]\nclass = \"anthropic\"\n" +
"api_key_env = \"ANTHROPIC_API_KEY\"\n",
overlay: "[providers.anthropic]\nbase_url = \"https://proxy.example\"\n",
check: func(t *testing.T, cfg config.Config) {
t.Helper()
got := cfg.Providers["anthropic"]
want := config.Provider{
Class: "anthropic",
APIKeyEnv: "ANTHROPIC_API_KEY",
BaseURL: "https://proxy.example",
}
if got != want {
t.Errorf("providers.anthropic = %+v, want %+v", got, want)
}
},
},
{
// The overlay file may be YAML (-config-overlay accepts either
// format), and it is folded by the same code.
name: "yaml overlay keeps the token it does not name",
base: "[services.state]\ntarget = \"127.0.0.1:9090\"\ntarget_token = \"secret\"\n",
overlayName: "dev.yaml",
overlay: "services:\n state:\n target: 10.0.0.5:9090\n",
check: func(t *testing.T, cfg config.Config) {
t.Helper()
got := cfg.Services[config.ServiceNameState]
if got.Target != "10.0.0.5:9090" {
t.Errorf("services.state.target = %q, want the overlay's 10.0.0.5:9090", got.Target)
}
if got.TargetToken != "secret" {
t.Errorf("services.state.target_token = %q, want the base's secret", got.TargetToken)
}
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
dir := t.TempDir()
basePath := filepath.Join(dir, "config.toml")
overlayName := tt.overlayName
if overlayName == "" {
overlayName = "dev.toml"
}
overlayPath := filepath.Join(dir, overlayName)
if err := os.WriteFile(basePath, []byte(minimalValidConfigTOML+tt.base), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(overlayPath, []byte(tt.overlay), 0o600); err != nil {
t.Fatal(err)
}
doc, err := New(nil).Resolve(basePath, overlayPath)
if err != nil {
t.Fatalf("Resolve: %v", err)
}
tt.check(t, doc.Config)
})
}
}
// TestLoadAcceptsStaleDatabaseOwnedValues is the bootstrap half of the layer
// split (archie-core-i3qm). docs/prds/runtime-control-plane.md,
// "Bootstrap, migration, and recovery": after migration, settings in TOML are
// ignored and cannot block State Store startup. The other half is the boot
// itself -- archie-state-store seeds these same settings into the control plane
// on a fresh database, and a seed it cannot validate is skipped rather than fatal
// (TestStateStoreBootsWithStaleDatabaseOwnedValues boots the process for that).
//
// Each case is a setting with a control-plane resource behind it, seeded from
// that field by controlplane.Server.ImportConfig: providers
// (provider-settings), poll_interval and dispatch (scheduling-policy),
// containers (container-runtime-policies), repositories
// (repository-policies). The second assertion is what keeps this from being a
// weaker gate: the value is still rejected by Validate, which is what the daemon
// and the Gateway run over the document a process will actually use. Nothing in
// this package layers that document, so the check here is the file document on
// its own -- the test is that the loader judges no database-owned field, not
// that the value is harmless.
func TestLoadAcceptsStaleDatabaseOwnedValues(t *testing.T) {
tests := []struct {
name string
body string
}{
{
name: "providers base_url carries userinfo",
body: fileConfigPrefix + "[providers.openai]\nclass = \"openai\"\nbase_url = \"https://token@example.com/v1\"\n",
},
{
name: "negative poll_interval",
body: fileConfigPrefix + "poll_interval = \"-5s\"\n",
},
{
name: "unrecognised dispatch.trigger",
body: fileConfigPrefix + "[dispatch]\ntrigger = \"labels\"\n",
},
{
name: "negative containers.volume_ttl",
body: fileConfigPrefix + "[containers]\nvolume_ttl = \"-1m\"\n",
},
{
name: "malformed repos test_glob",
body: fileConfigPrefix + "[[repos]]\nowner = \"acme\"\nname = \"app\"\ntest_glob = \"[\"\n",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
path := filepath.Join(t.TempDir(), "config.toml")
if err := os.WriteFile(path, []byte(tt.body), 0o600); err != nil {
t.Fatal(err)
}
doc, err := New(nil).File(path)
if err != nil {
t.Fatalf("File: %v (a stale database-owned value must not fail the load)", err)
}
if err := Validate(&doc.Config); err == nil {
t.Fatal("Validate = nil, want the control plane's own layer to reject it")
}
})
}
}
// fileConfigPrefix is the smallest document the load path accepts on its own,
// so each case above adds one stale setting and nothing else.
const fileConfigPrefix = "bot_user = \"widget\"\nwork_dir = \"/base/work\"\n"
// TestDirOverlayPreservesOmittedMapEntryFields is the directory form of the
// same guarantee TestResolveFileOverlayPreservesOmittedMapEntryFields pins.
// -config-overlay accepts a directory as well as a file, Loader.Dir documents
// "the same field-level precedence as [Loader.Overlay]", and Loader.Resolve
// routes a directory source to Dir -- so a map-valued entry the overlay only
// partly addresses must keep the fields it does not name on both forms of the
// flag, including when the entry arrives through a feature file.
func TestDirOverlayPreservesOmittedMapEntryFields(t *testing.T) {
tests := []struct {
name string
baseFiles map[string]string
overlayFiles map[string]string
check func(t *testing.T, cfg config.Config)
}{
{
name: "main config keeps the token it does not name",
baseFiles: map[string]string{
"config.yaml": "bot_user: widget\nservices:\n state:\n target: 127.0.0.1:9090\n target_token: secret\n",
},
overlayFiles: map[string]string{
"config.yaml": "services:\n state:\n target: 10.0.0.5:9090\n",
},
check: func(t *testing.T, cfg config.Config) {
t.Helper()
got := cfg.Services[config.ServiceNameState]
if got.Target != "10.0.0.5:9090" {
t.Errorf("services.state.target = %q, want the overlay's 10.0.0.5:9090", got.Target)
}
if got.TargetToken != "secret" {
t.Errorf("services.state.target_token = %q, want the base's secret: a directory overlay must not clear a field it does not name", got.TargetToken)
}
},
},
{
name: "feature file keeps the fields it does not name",
baseFiles: map[string]string{
"config.yaml": "bot_user: widget\n",
"config.models.yaml": "providers:\n anthropic:\n class: anthropic\n api_key_env: ANTHROPIC_API_KEY\n",
},
overlayFiles: map[string]string{
"config.models.yaml": "providers:\n anthropic:\n base_url: https://proxy.example\n",
},
check: func(t *testing.T, cfg config.Config) {
t.Helper()
got := cfg.Providers["anthropic"]
want := config.Provider{
Class: "anthropic",
APIKeyEnv: "ANTHROPIC_API_KEY",
BaseURL: "https://proxy.example",
}
if got != want {
t.Errorf("providers.anthropic = %+v, want %+v", got, want)
}
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
baseDir, overlayDir := tmpConfigDir(t), tmpConfigDir(t)
for name, content := range tt.baseFiles {
writeFile(t, baseDir, name, content)
}
for name, content := range tt.overlayFiles {
writeFile(t, overlayDir, name, content)
}
doc, err := New(nil).Resolve(baseDir, overlayDir)
if err != nil {
t.Fatalf("Resolve: %v", err)
}
tt.check(t, doc.Config)
})
}
}
func TestLoadBytesDoesNotExist(t *testing.T) {
_, err := loadBytes(nil)
if err == nil {
@@ -89,6 +89,60 @@ func TestUnknownSchedulingKeyIsStillReported(t *testing.T) {
}
}
// TestOverlayUnknownKeysAreStillReported pins that the overlay path kept its
// drift detection when the overlay file started being applied through the
// config fold instead of being decoded into the document's config. That matters
// twice over there: the report comes from a separate decode, and a file is
// parsed by TOML while the fold applies it with a yaml decode, which do not
// match keys the same way. A key the APPLY cannot consume has to be reported,
// or a spelling that works in the base config would silently do nothing in the
// overlay.
func TestOverlayUnknownKeysAreStillReported(t *testing.T) {
tests := []struct {
name string
overlay string
wantUnknown string
}{
{
name: "a key no decode target consumes",
overlay: "[containers]\nmax_concurrancy = 4\n", // typo: concurrancy
wantUnknown: "containers.max_concurrancy",
},
{
// TOML matches keys case-insensitively, so the file's own decode
// consumes BOT_USER and reports nothing; the apply folds the file's
// mapping with a yaml decode, which does not match it. Without the
// second source for this report the value would be dropped in
// silence, so the spelling is reported instead.
name: "a key only the file decode can match",
overlay: "BOT_USER = \"upper\"\n",
wantUnknown: "BOT_USER",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
dir := t.TempDir()
basePath := filepath.Join(dir, "config.toml")
overlayPath := filepath.Join(dir, "dev.toml")
if err := os.WriteFile(basePath, []byte(minimalValidConfigTOML), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(overlayPath, []byte(tt.overlay), 0o600); err != nil {
t.Fatal(err)
}
doc, err := New(nil).Resolve(basePath, overlayPath)
if err != nil {
t.Fatalf("Resolve: %v (an unknown key must not fail the load)", err)
}
if !slices.Contains(doc.UnknownKeys, tt.wantUnknown) {
t.Errorf("UnknownKeys = %v, want it to contain %q", doc.UnknownKeys, tt.wantUnknown)
}
})
}
}
// TestExampleConfigHasNoUnknownKeys is the contract test the design
// promised: the checked-in template must never itself trip the detector.
func TestExampleConfigHasNoUnknownKeys(t *testing.T) {
+134 -28
View File
@@ -37,9 +37,23 @@ var (
memoryEngines = []string{memoryEngineBuiltin}
)
// Validate runs the same checks Loader applies before accepting a config,
// against a config.Config value already built in memory -- e.g. by archied
// setup, before it has written anything to disk.
// Validate judges a configuration a process will run with -- every check that
// can reject one -- against a config.Config value already built in memory, e.g.
// by archied setup before it has written anything to disk. It is a superset of
// what [Loader] applies to a file (validateBootstrap): a check over a setting
// the control plane owns belongs here and not there, so that a stale TOML value
// cannot fail a process's startup.
//
// It judges an EFFECTIVE document: the bootstrap file with every setting the
// control plane owns layered over it. That layering happens at exactly one
// place, boot.runtimeConfig (internal/app/archied/control_plane.go), which runs
// this afterwards; the daemon and the Gateway both go through it, and the
// readiness config probe re-runs it against the running value. A database value
// that will not validate therefore stops the process rather than starting it
// degraded.
//
// A plain file document is judged by validateBootstrap instead: see the split
// there for why the loader must not apply this to it.
//
// Like validate, it does not apply defaults. Several checks (dispatch.trigger,
// forge.type) only pass once a default has been filled
@@ -58,31 +72,37 @@ func Validate(cfg *config.Config) error {
// validate reports the first problem that would stop the daemon running.
// It does not modify cfg -- run applyDefaults first.
func validate(cfg *config.Config) error {
if err := validateDispatch(cfg); err != nil {
if err := validateBootstrap(cfg); err != nil {
return err
}
return validateDatabaseOwned(cfg)
}
// validateBootstrap reports the first problem in the settings a bootstrap
// document still owns: the file config a process reads before it can reach the
// State Store. [Loader] applies this to every file source.
//
// It deliberately omits the settings the control plane stores, because a stale
// TOML value in one of them must not be able to fail a process's startup
// (docs/prds/runtime-control-plane.md, "Bootstrap, migration, and recovery":
// after migration, settings in TOML are ignored and cannot block State Store
// startup). archie-state-store resolves its file config and seeds the
// control-plane resources from that same document on a fresh database, and a
// seed it cannot validate is skipped rather than fatal
// (controlplane.Server.ImportConfig), so a check left here is a check that can
// still block it. Those checks live in validateDatabaseOwned and run on the
// effective document instead -- which is where the process that uses the value
// refuses it, and where the operator's fix is the file again.
func validateBootstrap(cfg *config.Config) error {
if err := validateForgeIntake(cfg); err != nil {
return err
}
if err := validateProviders(cfg.Providers); err != nil {
return err
}
if len(cfg.Identities) > 0 {
if err := validateIdentities(cfg.Identities); err != nil {
return err
}
} else if err := validateSingleIdentity(cfg); err != nil {
return err
}
if err := validatePollInterval(cfg); err != nil {
if err := validateIdentityStructure(cfg); err != nil {
return err
}
if err := validateNATS(cfg); err != nil {
return err
}
if err := validateContainers(cfg); err != nil {
return err
}
if err := validateMemory(cfg); err != nil {
return err
}
@@ -92,6 +112,30 @@ func validate(cfg *config.Config) error {
return validateCapture(cfg)
}
// validateDatabaseOwned reports the first problem in a setting the control
// plane owns. Each check below judges a field that a control-plane resource is
// seeded from and replaces wholesale: providers (provider-settings),
// poll_interval and dispatch (scheduling-policy), containers
// (container-runtime-policies), and the repository lists
// (repository-policies). It runs on the effective document, after that
// replacement, which is the only point at which the value being judged is the
// value a process will use.
func validateDatabaseOwned(cfg *config.Config) error {
if err := validateDispatch(cfg); err != nil {
return err
}
if err := validateProviders(cfg.Providers); err != nil {
return err
}
if err := validatePollInterval(cfg); err != nil {
return err
}
if err := validateContainers(cfg); err != nil {
return err
}
return validateRepositoryContents(cfg)
}
// validateImage rejects an enabled hosted provider with no class or no
// resolvable credential, and a Default naming a provider that is not both
// present and enabled -- either would let /image silently fall back to
@@ -186,7 +230,7 @@ func validatePollInterval(cfg *config.Config) error {
// labels) was configured while the actual trigger-match label was left
// blank.
func validateDispatch(cfg *config.Config) error {
if !oneOf(cfg.Dispatch.Trigger, dispatchTriggers) {
if !DispatchTriggerValid(cfg.Dispatch.Trigger) {
return fmt.Errorf("%w: dispatch.trigger %q (want %s)", ErrInvalidInput, cfg.Dispatch.Trigger, list(dispatchTriggers))
}
if (cfg.Dispatch.Trigger == dispatchTriggerLabel || cfg.Dispatch.Trigger == dispatchTriggerEither) && cfg.Label == "" {
@@ -195,6 +239,16 @@ func validateDispatch(cfg *config.Config) error {
return nil
}
// DispatchTriggerValid reports whether trigger names a discovery rule the daemon
// can poll with. It is exported because the scheduling-policy resource validator
// (internal/app/controlplane) judges the same field: that resource is seeded
// from cfg.Dispatch and replaces it wholesale once the database owns it, so a
// value this package rejects must not be storable, and a value it accepts must
// not be rejected there. One definition, both layers.
func DispatchTriggerValid(trigger string) bool {
return oneOf(trigger, dispatchTriggers)
}
// validateForgeIntake checks the forge intake mode and that webhook intake has
// the secret and listen address it needs. An unset intake resolves to "poll"
// without mutating cfg, so Validate (which does not apply defaults) accepts a
@@ -237,6 +291,18 @@ func validateProviders(providers map[string]config.Provider) error {
return nil
}
// validateIdentityStructure judges the shape of the identity definitions: the
// fields a process needs to know which identities exist and which credentials
// each one commits and calls with. The repository lists inside them are judged
// by validateRepositoryContents, because repositories are a control-plane
// resource that replaces the file's list after the merge.
func validateIdentityStructure(cfg *config.Config) error {
if len(cfg.Identities) == 0 {
return validateSingleIdentity(cfg)
}
return validateIdentities(cfg.Identities)
}
func validateIdentities(identities []config.IdentityConfig) error {
for i, id := range identities {
if id.Name == "" {
@@ -254,9 +320,6 @@ func validateIdentities(identities []config.IdentityConfig) error {
if !ForgeDisabled(id.Forge.Type) && id.Forge.Token == (config.SecretRef{}) {
return fmt.Errorf("%w: identities[%d].forge.token is required (each identity needs its own secret reference; unlike the top-level [forge], there is no default)", ErrInvalidInput, i)
}
if err := validateRepos(id.Repos); err != nil {
return fmt.Errorf("identities[%d]: %w", i, err)
}
}
return nil
}
@@ -278,19 +341,62 @@ func validateSingleIdentity(cfg *config.Config) error {
if !oneOf(cfg.Forge.Type, forgeTypes) {
return fmt.Errorf("%w: forge.type %q (want %s)", ErrInvalidInput, cfg.Forge.Type, list(forgeTypes))
}
return validateRepos(cfg.Repos)
return nil
}
func validateRepos(repos []config.Repo) error {
// validateRepositoryContents judges the repositories themselves, whichever list
// they appear in. The shared list is the decision the database's copy replaces,
// so it is verified on the effective document -- after that replacement, which is
// the only point the value being judged is the one a process will use. The
// per-identity lists are verified there for a different reason: no resource
// carries them, and the daemon is the only process that reads them
// (docs/architecture/configuration.md's IdentityConfig note).
func validateRepositoryContents(cfg *config.Config) error {
if len(cfg.Identities) == 0 {
return ValidateRepositories(cfg.Repos)
}
for i, id := range cfg.Identities {
if err := ValidateRepositories(id.Repos); err != nil {
return fmt.Errorf("identities[%d]: %w", i, err)
}
}
return nil
}
// ValidateRepositories judges a repository list: owner and name present, no
// duplicate entries, and a test glob the gate can compile. It is exported
// because the repository-policies resource validator (internal/app/controlplane)
// judges the same list -- that resource is seeded from these and replaces them
// wholesale, so the two layers must not disagree about which lists are valid, in
// either direction: a list one layer accepts and the other refuses is a config
// that boots file-side and stores nowhere, or a stored value the daemon then
// refuses to start with.
func ValidateRepositories(repos []config.Repo) error {
seen := make(map[string]struct{}, len(repos))
for i, r := range repos {
if r.Owner == "" || r.Name == "" {
return fmt.Errorf("%w: repos[%d] needs owner and name", ErrInvalidInput, i)
}
if glob := r.ResolvedTestGlob(); glob != "" {
if _, err := filepath.Match(glob, ""); err != nil {
return fmt.Errorf("%w: repos[%d] test_glob %q: %w", ErrInvalidInput, i, glob, err)
}
if _, exists := seen[r.FullName()]; exists {
return fmt.Errorf("%w: repos[%d] duplicates %q", ErrInvalidInput, i, r.FullName())
}
seen[r.FullName()] = struct{}{}
if err := validateTestGlob(r.ResolvedTestGlob()); err != nil {
return fmt.Errorf("%w: repos[%d] %w", ErrInvalidInput, i, err)
}
}
return nil
}
// validateTestGlob reports whether glob is a pattern the test-protection gate can
// compile. It has one caller on each side through ValidateRepositories, so the
// rule has one definition rather than a copy per layer.
func validateTestGlob(glob string) error {
if glob == "" {
return nil
}
if _, err := filepath.Match(glob, ""); err != nil {
return fmt.Errorf("test_glob %q: %w", glob, err)
}
return nil
}
@@ -69,6 +69,12 @@ func TestValidate_IsCallableOutsideThePackage(t *testing.T) {
}
}
// TestValidate_RejectsTheSameProblemsAsLoaderLoad pins the direction that still
// holds after archie-core-i3qm split the bootstrap checks from the effective
// ones: every problem the load path rejects, Validate rejects too. Validate
// additionally rejects the settings the control plane owns, which the load path
// must let through (validateBootstrap) so a stale TOML value cannot fail a
// process's startup. The name is kept from when the two check sets were equal.
func TestValidate_RejectsTheSameProblemsAsLoaderLoad(t *testing.T) {
tests := []struct {
name string