From 0070fba7a21cec7accdec7bdfbf1b953d0474393 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Mon, 21 Sep 2026 20:21:33 +1000 Subject: [PATCH] fix(archied): probe what the live settings update would actually run Review of d7f3d0da found the build-and-probe step shape-only: the check it added re-ran workflow.ExecutionSettings.Validate, which the store runs before writing the document and controlplane.Client runs again on the way out, so no producer could trip it. That review was right; this corrects it. - The probe is now this process's own runnability check on the candidate it would switch to: configuration.Validate over the configuration snapshot the new limits would be published as, the same gate runtimeConfig applies before publishing a layered config and the readiness probe applies to the published one. The duplicated schema check is gone. - The refusal the daemon really meets arrives as a watch error, because the client decodes every document before the apply path sees it. That branch logged and returned without writing an apply-status record, so the settings page showed a live resource as healthy while nothing was being applied. It now reports the refusal, against the version the reporter last applied. Whether the stream is re-established afterwards stays archie-core-yrmr. - `Closes archie-core-nwa0` in d7f3d0da is withdrawn: the item stays open. For a limits-only kind no producer can make the owned check fail today, so the refusal path is exercised by test, not by traffic; the live kind whose candidate check has content is personas (archie-core-nw7t). Both callers are covered by tests now, through the client seam, instead of by calling the apply function directly. - The comments no longer claim behaviour the code lacks: the boot abort is the fail-closed direction of a rule whose branch boot cannot reach, and the carried-forward version is the reporter's property, not this call's. Out of scope and recorded instead: the live apply still publishes with cfgHolder.Set rather than publishConfig, leaving the dashboard's config snapshot stale after a live change (archie-core-75cc). workflow-definitions gets no check because nothing swaps a running definition: the daemon reads the collection per task pin (daemon.go:1479, pinWorkflowFromCollection) and the only other reader is the dashboard's stream, so there is no live replacement to probe. --- internal/app/archied/control_plane.go | 53 ++-- .../archied/control_plane_live_apply_test.go | 288 +++++++++++++++--- .../app/archied/control_plane_reload_test.go | 5 +- 3 files changed, 278 insertions(+), 68 deletions(-) diff --git a/internal/app/archied/control_plane.go b/internal/app/archied/control_plane.go index ac87b02be..4ddff2564 100644 --- a/internal/app/archied/control_plane.go +++ b/internal/app/archied/control_plane.go @@ -100,12 +100,20 @@ func (b *boot) startWorkflowExecutionSettings(ctx context.Context) error { go func() { for update := range updates { if update.Err != nil { + // A refused update arrives here, not through + // applyWorkflowExecutionSettings: controlplane.Client decodes every + // document it is handed, so a stored value this process cannot run + // never reaches the apply call. Report it against the version it + // came from -- the reporter keeps the version it last applied -- + // so the settings page shows the refusal instead of a version this + // process never ran (docs/prds/control-plane-apply-status.md). + // + // The stream is not re-established after this point; that gap is + // archie-core-yrmr, not this apply path. + b.applyStatus.Report(ctx, controlplane.WorkflowExecutionSettingsKind, update.Version, update.Err) b.log.Error("workflow execution settings watch failed", "err", update.Err) return } - // A refused update leaves the running component in place and is - // already reported through apply status, so the watch keeps - // streaming instead of stopping on it. _ = b.applyWorkflowExecutionSettings(ctx, update.Settings, update.Version) } }() @@ -121,20 +129,25 @@ type executionSettingsCandidate struct { } // buildExecutionSettingsCandidate stages a live update without touching -// anything running: it checks the candidate the running limits would be -// replaced with, so a change this process cannot run is refused before it -// becomes the live one (docs/prds/runtime-control-plane.md, "API": Archie -// starts and checks the new one before switching). +// anything running: it builds the configuration snapshot the new limits would +// be published as and runs this process's own runnability check on it -- the +// same configuration.Validate that runtimeConfig applies before publishing a +// layered config, and that the readiness probe applies to the published one. A +// snapshot this process cannot run is refused before it becomes the live one +// (docs/prds/runtime-control-plane.md, "API": Archie starts and checks the new +// one before switching). // -// The check runs here rather than only where the document was written because -// a live update is promoted long after that: the store's validation describes -// what may be stored, and this is the process that has to run it. +// The kind's schema is deliberately not re-checked here. The store validates +// every document before it is written (Definition.Decode) and +// controlplane.Client validates it again as it decodes it, so re-running +// workflow.ExecutionSettings.Validate at this point would be a guard no +// producer can trip. func (b *boot) buildExecutionSettingsCandidate(settings workflow.ExecutionSettings) (executionSettingsCandidate, error) { - if err := settings.Validate(); err != nil { - return executionSettingsCandidate{}, fmt.Errorf("workflow execution settings: %w", err) - } cfg := b.cfgHolder.Get().Clone() applyExecutionBudgets(&cfg, settings) + if err := configuration.Validate(&cfg); err != nil { + return executionSettingsCandidate{}, fmt.Errorf("workflow execution settings: %w", err) + } return executionSettingsCandidate{settings: settings, cfg: cfg}, nil } @@ -143,16 +156,20 @@ func (b *boot) buildExecutionSettingsCandidate(settings workflow.ExecutionSettin // Both halves of the running component move together and only after the check // passes: on a refusal the previous settings keep running, the configuration // snapshot new tasks are built from is left as it is, and the failure is -// reported against the version that is still live, so the settings page shows -// a rejected edit rather than a version this process never ran. +// reported for the version that was refused. The apply-status reporter writes +// that record against the version it last applied, so once this process has +// applied anything the settings page shows a rejected edit rather than a +// version this process never ran. // // The settings are recorded as well as applied: they arrive on a watch rather // than in the file document, so a reload has nowhere else to read them back // from. // -// The returned error is for a caller that cannot continue on limits that were -// not applied -- boot, which must not start a process it cannot run. On the -// watch path the running component stays live and the report is the outcome. +// The returned error is propagated by boot, the caller that cannot continue on +// limits it did not apply. Boot reaches this function only with settings the +// control plane already decoded and over a snapshot loadRuntimeConfig already +// validated, so that propagation is the fail-closed direction of the rule +// above rather than a branch the daemon takes today. func (b *boot) applyWorkflowExecutionSettings(ctx context.Context, settings workflow.ExecutionSettings, version int64) error { candidate, err := b.buildExecutionSettingsCandidate(settings) if err != nil { diff --git a/internal/app/archied/control_plane_live_apply_test.go b/internal/app/archied/control_plane_live_apply_test.go index ba711d217..9f9e05467 100644 --- a/internal/app/archied/control_plane_live_apply_test.go +++ b/internal/app/archied/control_plane_live_apply_test.go @@ -2,18 +2,32 @@ package archied import ( "context" + "io" "log/slog" + "strings" "sync" "testing" "time" + "google.golang.org/grpc" + "github.com/samcharles93/archie-core/internal/app/controlplane" "github.com/samcharles93/archie-core/internal/config" + pb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1" "github.com/samcharles93/archie-core/internal/domain/applystatus" "github.com/samcharles93/archie-core/internal/domain/storecontract" "github.com/samcharles93/archie-core/internal/domain/workflow" ) +// The limits this package's fixtures boot or run with, and the documents the +// control plane stores them as. Named so a test that asserts "the running +// limits stayed" and a test that asserts "the update was applied" cannot +// disagree about what "running" means. +var ( + bootedSettings = workflow.ExecutionSettings{MaxModelToolSteps: 10, MaxRuntime: 5 * time.Minute, MaxConsecutiveGateFailures: 3} + bootedDocument = `{"max_model_tool_steps": 10, "max_runtime_seconds": 300, "max_consecutive_gate_failures": 3}` +) + // applyStatusRecorder is the State Store end of the apply-status surface: it // keeps what a process reported, so a test can assert what the settings page // would render. @@ -47,86 +61,144 @@ func (r *applyStatusRecorder) last(kind string) (storecontract.ApplyStatus, bool return storecontract.ApplyStatus{}, false } -// newLiveApplyBoot builds a boot whose live kinds report apply status through -// a recorder, so a test can read back what the settings page would show. -func newLiveApplyBoot(t *testing.T) (*boot, *applyStatusRecorder) { +// awaitCount waits until n records have been written for kind. The live apply +// runs on the watch goroutine, so a test that drives it through the stream has +// nothing to synchronise on but the record it is waiting for. +func (r *applyStatusRecorder) awaitCount(t *testing.T, kind string, n int) []storecontract.ApplyStatus { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for { + r.mu.Lock() + written := make([]storecontract.ApplyStatus, 0, n) + for _, status := range r.writes { + if status.Kind == kind { + written = append(written, status) + } + } + r.mu.Unlock() + if len(written) >= n { + return written + } + if time.Now().After(deadline) { + t.Fatalf("%s: %d records reported, want %d", kind, len(written), n) + } + time.Sleep(5 * time.Millisecond) + } +} + +// newLiveApplyBoot builds a boot running the given snapshot, whose apply status +// reports land in a recorder. +func newLiveApplyBoot(t *testing.T, running config.Config) (*boot, *applyStatusRecorder) { t.Helper() recorder := &applyStatusRecorder{} b := &boot{ - cfg: fileConfig(), + cfg: running, log: slog.New(slog.DiscardHandler), - cfgHolder: config.NewHolder(fileConfig()), + cfgHolder: config.NewHolder(running), processName: applystatus.Daemon, } b.applyStatus = applystatus.New(b.processName, recorder, b.log) return b, recorder } +// settingsWatchStub answers Query with one stored settings document and Watch +// with a stream carrying the given documents. It is the seam the daemon really +// meets a live update through: the control plane stores documents, and +// controlplane.Client decodes them before the apply path sees anything. +type settingsWatchStub struct { + stored string + version int64 + documents []string +} + +func (*settingsWatchStub) Catalog(context.Context, *pb.CatalogRequest, ...grpc.CallOption) (*pb.CatalogResponse, error) { + return &pb.CatalogResponse{}, nil +} + +func (s *settingsWatchStub) Query(context.Context, *pb.QueryRequest, ...grpc.CallOption) (*pb.QueryResponse, error) { + return &pb.QueryResponse{Resource: &pb.Resource{Kind: controlplane.WorkflowExecutionSettingsKind, Version: s.version, ValueJson: []byte(s.stored)}}, nil +} + +func (*settingsWatchStub) History(context.Context, *pb.HistoryRequest, ...grpc.CallOption) (*pb.HistoryResponse, error) { + panic("unexpected History") +} + +func (*settingsWatchStub) Command(context.Context, *pb.CommandRequest, ...grpc.CallOption) (*pb.CommandResponse, error) { + panic("unexpected Command") +} + +func (s *settingsWatchStub) Watch(context.Context, *pb.WatchRequest, ...grpc.CallOption) (grpc.ServerStreamingClient[pb.WatchResponse], error) { + responses := make([]*pb.WatchResponse, 0, len(s.documents)) + for i, document := range s.documents { + responses = append(responses, &pb.WatchResponse{Resource: &pb.Resource{ + Kind: controlplane.WorkflowExecutionSettingsKind, + Version: s.version + int64(i) + 1, + ValueJson: []byte(document), + }}) + } + return &watchStreamStub{responses: responses}, nil +} + +// watchStreamStub is the client end of a Watch stream. Recv is the only method +// the settings watch calls; the embedded ClientStream is nil, so any other call +// panics rather than pretending to work. +type watchStreamStub struct { + grpc.ClientStream + responses []*pb.WatchResponse + next int +} + +func (s *watchStreamStub) Recv() (*pb.WatchResponse, error) { + if s.next >= len(s.responses) { + return nil, io.EOF + } + response := s.responses[s.next] + s.next++ + return response, nil +} + // TestLiveExecutionSettingsUpdateIsCheckedBeforeItReplacesTheRunningLimits is // archie-core-nwa0. docs/prds/runtime-control-plane.md ("API") requires a live // change to replace a running component only after the new one has been // started and checked; if the check fails the old one keeps running. A live -// workflow-execution-settings update replaces the limits every new task is -// built from, so a candidate the component refuses must leave the running -// limits and the running configuration snapshot in place, and the rejection -// must reach the apply-status surface carrying the version that is still live. +// workflow-execution-settings update replaces the configuration snapshot every +// new task is built from, so a candidate whose snapshot this process cannot run +// must leave the running limits and the running snapshot in place, and the +// failure must reach the apply-status surface carrying the version that is +// still live. func TestLiveExecutionSettingsUpdateIsCheckedBeforeItReplacesTheRunningLimits(t *testing.T) { - // The limits the daemon is already running when the update arrives. - running := workflow.ExecutionSettings{MaxModelToolSteps: 10, MaxRuntime: 5 * time.Minute, MaxConsecutiveGateFailures: 3} - tests := []struct { name string candidate workflow.ExecutionSettings wantRunning workflow.ExecutionSettings - wantError bool wantVersion int64 }{ { - name: "an update the component accepts replaces the running limits", + name: "an update whose snapshot this process can run replaces the running limits", candidate: workflow.ExecutionSettings{MaxModelToolSteps: 25, MaxRuntime: time.Hour, MaxConsecutiveGateFailures: 4}, wantRunning: workflow.ExecutionSettings{MaxModelToolSteps: 25, MaxRuntime: time.Hour, MaxConsecutiveGateFailures: 4}, wantVersion: 2, }, { // Zero is the documented "this individual limit is off", so an - // all-zero update is a legal live change, not a rejection. + // all-zero update is a legal live change, not a rejection, and the + // check must not invent a bound the store does not enforce. name: "an update that disables every limit is accepted", candidate: workflow.ExecutionSettings{}, wantRunning: workflow.ExecutionSettings{}, wantVersion: 2, }, - { - name: "an update with a negative step limit is refused and the running limits stay", - candidate: workflow.ExecutionSettings{MaxModelToolSteps: -1, MaxRuntime: time.Hour, MaxConsecutiveGateFailures: 4}, - wantRunning: running, - wantError: true, - wantVersion: 1, - }, - { - name: "an update with a negative runtime is refused and the running limits stay", - candidate: workflow.ExecutionSettings{MaxModelToolSteps: 25, MaxRuntime: -time.Second, MaxConsecutiveGateFailures: 4}, - wantRunning: running, - wantError: true, - wantVersion: 1, - }, - { - name: "an update with a negative gate-failure limit is refused and the running limits stay", - candidate: workflow.ExecutionSettings{MaxModelToolSteps: 25, MaxRuntime: time.Hour, MaxConsecutiveGateFailures: -1}, - wantRunning: running, - wantError: true, - wantVersion: 1, - }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - b, status := newLiveApplyBoot(t) - if err := b.applyWorkflowExecutionSettings(t.Context(), running, 1); err != nil { + b, status := newLiveApplyBoot(t, fileConfig()) + if err := b.applyWorkflowExecutionSettings(t.Context(), bootedSettings, 1); err != nil { t.Fatalf("applying the running settings: %v", err) } - if err := b.applyWorkflowExecutionSettings(t.Context(), tt.candidate, 2); (err != nil) != tt.wantError { - t.Fatalf("live update error = %v, want an error: %v", err, tt.wantError) + if err := b.applyWorkflowExecutionSettings(t.Context(), tt.candidate, 2); err != nil { + t.Fatalf("live update: %v", err) } got := b.executionSettings.Load() @@ -137,13 +209,9 @@ func TestLiveExecutionSettingsUpdateIsCheckedBeforeItReplacesTheRunningLimits(t t.Errorf("running settings = %+v, want %+v", *got, tt.wantRunning) } - // The snapshot a new task is built from must move with the - // settings, and must not move for a refused update. - wantBudgets := config.Budgets{ - MaxSteps: tt.wantRunning.MaxModelToolSteps, - WallClock: config.Duration(tt.wantRunning.MaxRuntime), - GateMaxFailures: tt.wantRunning.MaxConsecutiveGateFailures, - } + // The snapshot a new task is built from has to move with the + // settings: it is the half of the component the daemon runs. + wantBudgets := budgetsFor(tt.wantRunning) if budgets := b.cfgHolder.Get().Budgets; budgets != wantBudgets { t.Errorf("running budgets = %+v, want %+v", budgets, wantBudgets) } @@ -155,9 +223,131 @@ func TestLiveExecutionSettingsUpdateIsCheckedBeforeItReplacesTheRunningLimits(t if last.AppliedVersion != tt.wantVersion { t.Errorf("apply status version = %d, want %d", last.AppliedVersion, tt.wantVersion) } - if (last.Error != "") != tt.wantError { - t.Errorf("apply status error = %q, want an error: %v", last.Error, tt.wantError) + if last.Error != "" { + t.Errorf("apply status error = %q, want none for an applied update", last.Error) } }) } } + +// TestLiveExecutionSettingsUpdateIsRefusedWhenTheSnapshotIsNotRunnable covers +// the refusal half of the same check. The candidate is the running snapshot +// with new limits written into it, and configuration.Validate does not read +// limits, so the only input that can fail the check is a snapshot this process +// would not run -- which is what this test's daemon is running. A live update +// must not publish a snapshot this process cannot run, must leave the component +// it would have replaced in place, and must report the refusal against the +// version still live. +func TestLiveExecutionSettingsUpdateIsRefusedWhenTheSnapshotIsNotRunnable(t *testing.T) { + unrunnable := fileConfig() + unrunnable.Containers.Image = "" + applyExecutionBudgets(&unrunnable, bootedSettings) + b, status := newLiveApplyBoot(t, unrunnable) + // The daemon is already running these limits. Seeded directly rather than + // applied: the ordinary apply refuses this snapshot too, which is the point + // of the check. + b.executionSettings.Store(&bootedSettings) + b.applyStatus.Report(t.Context(), controlplane.WorkflowExecutionSettingsKind, 1, nil) + + budgets := budgetsFor(bootedSettings) + update := workflow.ExecutionSettings{MaxModelToolSteps: 25, MaxRuntime: time.Hour, MaxConsecutiveGateFailures: 4} + if err := b.applyWorkflowExecutionSettings(t.Context(), update, 2); err == nil { + t.Fatal("an update whose snapshot this process cannot run was applied") + } + + if got := b.executionSettings.Load(); got == nil || *got != bootedSettings { + t.Errorf("running settings = %v, want the refused update to leave %+v running", got, bootedSettings) + } + if got := b.cfgHolder.Get().Budgets; got != budgets { + t.Errorf("running budgets = %+v, want the refused update to leave %+v running", got, budgets) + } + + last, ok := status.last(controlplane.WorkflowExecutionSettingsKind) + if !ok { + t.Fatal("the refused update reported no apply status") + } + if last.Error == "" { + t.Error("the refusal was not reported through apply status") + } + if last.AppliedVersion != 1 { + t.Errorf("apply status version = %d, want the version still live (1)", last.AppliedVersion) + } +} + +// TestLiveExecutionSettingsRefusalIsReportedThroughTheClient is the other half +// of the same item, over the seam the daemon actually meets a refusal through. +// controlplane.Client decodes every document it is handed, so an update this +// process cannot run arrives as a watch error, never as invalid settings: the +// watch branch must report that refusal against the version still live instead +// of only logging it, or the settings page shows a live resource as healthy +// while nothing has been applied. (That the stream is not re-established +// afterwards is archie-core-yrmr, not this apply path.) +func TestLiveExecutionSettingsRefusalIsReportedThroughTheClient(t *testing.T) { + tests := []struct { + name string + document string + wantRunning workflow.ExecutionSettings + wantVersion int64 + wantErrorContains string + }{ + { + name: "an update the process can run is applied", + document: `{"max_model_tool_steps": 25, "max_runtime_seconds": 3600, "max_consecutive_gate_failures": 4}`, + wantRunning: workflow.ExecutionSettings{MaxModelToolSteps: 25, MaxRuntime: time.Hour, MaxConsecutiveGateFailures: 4}, + wantVersion: 2, + }, + { + name: "an update the client refuses is reported against the version still live", + document: `{"max_model_tool_steps": -1, "max_runtime_seconds": 3600, "max_consecutive_gate_failures": 4}`, + wantRunning: bootedSettings, + wantVersion: 1, + wantErrorContains: "max model/tool steps must not be negative", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + b, status := newLiveApplyBoot(t, fileConfig()) + b.controlPlane = controlplane.NewRPCClient(&settingsWatchStub{stored: bootedDocument, version: 1, documents: []string{tt.document}}) + + if err := b.startWorkflowExecutionSettings(t.Context()); err != nil { + t.Fatalf("startWorkflowExecutionSettings: %v", err) + } + // One record for boot, one for the watched update. + records := status.awaitCount(t, controlplane.WorkflowExecutionSettingsKind, 2) + + got := b.executionSettings.Load() + if got == nil { + t.Fatal("no settings are recorded as running") + } + if *got != tt.wantRunning { + t.Errorf("running settings = %+v, want %+v", *got, tt.wantRunning) + } + if budgets := b.cfgHolder.Get().Budgets; budgets != budgetsFor(tt.wantRunning) { + t.Errorf("running budgets = %+v, want %+v", budgets, budgetsFor(tt.wantRunning)) + } + + last := records[len(records)-1] + if last.AppliedVersion != tt.wantVersion { + t.Errorf("apply status version = %d, want %d", last.AppliedVersion, tt.wantVersion) + } + if tt.wantErrorContains == "" && last.Error != "" { + t.Errorf("apply status error = %q, want none for an applied update", last.Error) + } + if tt.wantErrorContains != "" && !strings.Contains(last.Error, tt.wantErrorContains) { + t.Errorf("apply status error = %q, want it to carry %q", last.Error, tt.wantErrorContains) + } + }) + } +} + +// budgetsFor is the configuration snapshot a set of limits produces. Named so +// the tests and applyExecutionBudgets cannot drift on the mapping. +func budgetsFor(settings workflow.ExecutionSettings) config.Budgets { + return config.Budgets{ + MaxSteps: settings.MaxModelToolSteps, + WallClock: config.Duration(settings.MaxRuntime), + GateMaxFailures: settings.MaxConsecutiveGateFailures, + } +} diff --git a/internal/app/archied/control_plane_reload_test.go b/internal/app/archied/control_plane_reload_test.go index 6ee0b4736..d2ecee6c3 100644 --- a/internal/app/archied/control_plane_reload_test.go +++ b/internal/app/archied/control_plane_reload_test.go @@ -65,7 +65,9 @@ func databaseOwnedResources() map[string]any { } // fileConfig is what the loader produces from config.toml alone: the layer a -// SIGHUP reload re-resolves, holding none of the database's values. +// SIGHUP reload re-resolves, holding none of the database's values. It is a +// config the daemon would run, so it passes configuration.Validate -- the live +// apply path runs that same check over the snapshot it is about to publish. func fileConfig() config.Config { return config.Config{ BotUser: "widget", @@ -79,6 +81,7 @@ func fileConfig() config.Config { PluginDir: "/file/plugins", Containers: config.ContainerConfig{Image: "archie:from-file", PullPolicy: "missing"}, Budgets: config.Budgets{MaxSteps: 1, WallClock: config.Duration(time.Minute)}, + Dispatch: config.Dispatch{Trigger: "assignee"}, } }