diff --git a/internal/app/archied/control_plane.go b/internal/app/archied/control_plane.go index ad78b4a66..ac87b02be 100644 --- a/internal/app/archied/control_plane.go +++ b/internal/app/archied/control_plane.go @@ -90,7 +90,9 @@ func (b *boot) startWorkflowExecutionSettings(ctx context.Context) error { if err != nil { return err } - b.applyWorkflowExecutionSettings(ctx, settings, version) + if err := b.applyWorkflowExecutionSettings(ctx, settings, version); err != nil { + return fmt.Errorf("apply workflow execution settings: %w", err) + } updates, err := b.controlPlane.WatchWorkflowExecutionSettings(ctx, version) if err != nil { return err @@ -101,22 +103,68 @@ func (b *boot) startWorkflowExecutionSettings(ctx context.Context) error { b.log.Error("workflow execution settings watch failed", "err", update.Err) return } - b.applyWorkflowExecutionSettings(ctx, update.Settings, update.Version) + // A refused update leaves the running component in place and is + // already reported through apply status, so the watch keeps + // streaming instead of stopping on it. + _ = b.applyWorkflowExecutionSettings(ctx, update.Settings, update.Version) } }() return nil } -// applyWorkflowExecutionSettings records the settings as well as applying -// them: they arrive on a watch rather than in the file document, so a reload -// has nowhere else to read them back from. -func (b *boot) applyWorkflowExecutionSettings(ctx context.Context, settings workflow.ExecutionSettings, version int64) { - b.executionSettings.Store(&settings) +// executionSettingsCandidate is a live workflow-execution-settings update that +// has been built and checked but not yet switched in: the limits themselves, +// and the configuration snapshot every new task would be built from. +type executionSettingsCandidate struct { + settings workflow.ExecutionSettings + cfg config.Config +} + +// buildExecutionSettingsCandidate stages a live update without touching +// anything running: it checks the candidate the running limits would be +// replaced with, so a change this process cannot run is refused before it +// becomes the live one (docs/prds/runtime-control-plane.md, "API": Archie +// starts and checks the new one before switching). +// +// The check runs here rather than only where the document was written because +// a live update is promoted long after that: the store's validation describes +// what may be stored, and this is the process that has to run it. +func (b *boot) buildExecutionSettingsCandidate(settings workflow.ExecutionSettings) (executionSettingsCandidate, error) { + if err := settings.Validate(); err != nil { + return executionSettingsCandidate{}, fmt.Errorf("workflow execution settings: %w", err) + } cfg := b.cfgHolder.Get().Clone() applyExecutionBudgets(&cfg, settings) - b.cfgHolder.Set(cfg) + return executionSettingsCandidate{settings: settings, cfg: cfg}, nil +} + +// applyWorkflowExecutionSettings builds a candidate and switches it in. +// +// Both halves of the running component move together and only after the check +// passes: on a refusal the previous settings keep running, the configuration +// snapshot new tasks are built from is left as it is, and the failure is +// reported against the version that is still live, so the settings page shows +// a rejected edit rather than a version this process never ran. +// +// The settings are recorded as well as applied: they arrive on a watch rather +// than in the file document, so a reload has nowhere else to read them back +// from. +// +// The returned error is for a caller that cannot continue on limits that were +// not applied -- boot, which must not start a process it cannot run. On the +// watch path the running component stays live and the report is the outcome. +func (b *boot) applyWorkflowExecutionSettings(ctx context.Context, settings workflow.ExecutionSettings, version int64) error { + candidate, err := b.buildExecutionSettingsCandidate(settings) + if err != nil { + b.applyStatus.Report(ctx, controlplane.WorkflowExecutionSettingsKind, version, err) + b.log.Error("workflow execution settings rejected; the running settings stay", "version", version, "err", err) + return err + } + b.executionSettings.Store(&candidate.settings) + b.cfgHolder.Set(candidate.cfg) b.applyStatus.Report(ctx, controlplane.WorkflowExecutionSettingsKind, version, nil) b.log.Info("workflow execution settings applied", "version", version) + return nil } func applyExecutionBudgets(cfg *config.Config, settings workflow.ExecutionSettings) { diff --git a/internal/app/archied/control_plane_live_apply_test.go b/internal/app/archied/control_plane_live_apply_test.go new file mode 100644 index 000000000..ba711d217 --- /dev/null +++ b/internal/app/archied/control_plane_live_apply_test.go @@ -0,0 +1,163 @@ +package archied + +import ( + "context" + "log/slog" + "sync" + "testing" + "time" + + "github.com/samcharles93/archie-core/internal/app/controlplane" + "github.com/samcharles93/archie-core/internal/config" + "github.com/samcharles93/archie-core/internal/domain/applystatus" + "github.com/samcharles93/archie-core/internal/domain/storecontract" + "github.com/samcharles93/archie-core/internal/domain/workflow" +) + +// applyStatusRecorder is the State Store end of the apply-status surface: it +// keeps what a process reported, so a test can assert what the settings page +// would render. +type applyStatusRecorder struct { + mu sync.Mutex + writes []storecontract.ApplyStatus +} + +func (r *applyStatusRecorder) PutApplyStatus(_ context.Context, status storecontract.ApplyStatus) error { + r.mu.Lock() + defer r.mu.Unlock() + r.writes = append(r.writes, status) + return nil +} + +func (r *applyStatusRecorder) ListApplyStatus(context.Context) ([]storecontract.ApplyStatus, error) { + r.mu.Lock() + defer r.mu.Unlock() + return append([]storecontract.ApplyStatus(nil), r.writes...), nil +} + +// last returns the most recent record written for kind. +func (r *applyStatusRecorder) last(kind string) (storecontract.ApplyStatus, bool) { + r.mu.Lock() + defer r.mu.Unlock() + for i := len(r.writes) - 1; i >= 0; i-- { + if r.writes[i].Kind == kind { + return r.writes[i], true + } + } + return storecontract.ApplyStatus{}, false +} + +// newLiveApplyBoot builds a boot whose live kinds report apply status through +// a recorder, so a test can read back what the settings page would show. +func newLiveApplyBoot(t *testing.T) (*boot, *applyStatusRecorder) { + t.Helper() + recorder := &applyStatusRecorder{} + b := &boot{ + cfg: fileConfig(), + log: slog.New(slog.DiscardHandler), + cfgHolder: config.NewHolder(fileConfig()), + processName: applystatus.Daemon, + } + b.applyStatus = applystatus.New(b.processName, recorder, b.log) + return b, recorder +} + +// TestLiveExecutionSettingsUpdateIsCheckedBeforeItReplacesTheRunningLimits is +// archie-core-nwa0. docs/prds/runtime-control-plane.md ("API") requires a live +// change to replace a running component only after the new one has been +// started and checked; if the check fails the old one keeps running. A live +// workflow-execution-settings update replaces the limits every new task is +// built from, so a candidate the component refuses must leave the running +// limits and the running configuration snapshot in place, and the rejection +// must reach the apply-status surface carrying the version that is still live. +func TestLiveExecutionSettingsUpdateIsCheckedBeforeItReplacesTheRunningLimits(t *testing.T) { + // The limits the daemon is already running when the update arrives. + running := workflow.ExecutionSettings{MaxModelToolSteps: 10, MaxRuntime: 5 * time.Minute, MaxConsecutiveGateFailures: 3} + + tests := []struct { + name string + candidate workflow.ExecutionSettings + wantRunning workflow.ExecutionSettings + wantError bool + wantVersion int64 + }{ + { + name: "an update the component accepts replaces the running limits", + candidate: workflow.ExecutionSettings{MaxModelToolSteps: 25, MaxRuntime: time.Hour, MaxConsecutiveGateFailures: 4}, + wantRunning: workflow.ExecutionSettings{MaxModelToolSteps: 25, MaxRuntime: time.Hour, MaxConsecutiveGateFailures: 4}, + wantVersion: 2, + }, + { + // Zero is the documented "this individual limit is off", so an + // all-zero update is a legal live change, not a rejection. + name: "an update that disables every limit is accepted", + candidate: workflow.ExecutionSettings{}, + wantRunning: workflow.ExecutionSettings{}, + wantVersion: 2, + }, + { + name: "an update with a negative step limit is refused and the running limits stay", + candidate: workflow.ExecutionSettings{MaxModelToolSteps: -1, MaxRuntime: time.Hour, MaxConsecutiveGateFailures: 4}, + wantRunning: running, + wantError: true, + wantVersion: 1, + }, + { + name: "an update with a negative runtime is refused and the running limits stay", + candidate: workflow.ExecutionSettings{MaxModelToolSteps: 25, MaxRuntime: -time.Second, MaxConsecutiveGateFailures: 4}, + wantRunning: running, + wantError: true, + wantVersion: 1, + }, + { + name: "an update with a negative gate-failure limit is refused and the running limits stay", + candidate: workflow.ExecutionSettings{MaxModelToolSteps: 25, MaxRuntime: time.Hour, MaxConsecutiveGateFailures: -1}, + wantRunning: running, + wantError: true, + wantVersion: 1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + b, status := newLiveApplyBoot(t) + if err := b.applyWorkflowExecutionSettings(t.Context(), running, 1); err != nil { + t.Fatalf("applying the running settings: %v", err) + } + if err := b.applyWorkflowExecutionSettings(t.Context(), tt.candidate, 2); (err != nil) != tt.wantError { + t.Fatalf("live update error = %v, want an error: %v", err, tt.wantError) + } + + got := b.executionSettings.Load() + if got == nil { + t.Fatal("no settings are recorded as running") + } + if *got != tt.wantRunning { + t.Errorf("running settings = %+v, want %+v", *got, tt.wantRunning) + } + + // The snapshot a new task is built from must move with the + // settings, and must not move for a refused update. + wantBudgets := config.Budgets{ + MaxSteps: tt.wantRunning.MaxModelToolSteps, + WallClock: config.Duration(tt.wantRunning.MaxRuntime), + GateMaxFailures: tt.wantRunning.MaxConsecutiveGateFailures, + } + if budgets := b.cfgHolder.Get().Budgets; budgets != wantBudgets { + t.Errorf("running budgets = %+v, want %+v", budgets, wantBudgets) + } + + last, ok := status.last(controlplane.WorkflowExecutionSettingsKind) + if !ok { + t.Fatal("the live update reported no apply status") + } + if last.AppliedVersion != tt.wantVersion { + t.Errorf("apply status version = %d, want %d", last.AppliedVersion, tt.wantVersion) + } + if (last.Error != "") != tt.wantError { + t.Errorf("apply status error = %q, want an error: %v", last.Error, tt.wantError) + } + }) + } +} diff --git a/internal/app/archied/control_plane_reload_test.go b/internal/app/archied/control_plane_reload_test.go index 348db1e66..6ee0b4736 100644 --- a/internal/app/archied/control_plane_reload_test.go +++ b/internal/app/archied/control_plane_reload_test.go @@ -102,9 +102,11 @@ func TestReloadConfigKeepsDatabaseOwnedSettings(t *testing.T) { if err := b.loadRuntimeConfig(t.Context()); err != nil { t.Fatalf("loadRuntimeConfig: %v", err) } - b.applyWorkflowExecutionSettings(t.Context(), workflow.ExecutionSettings{ + if err := b.applyWorkflowExecutionSettings(t.Context(), workflow.ExecutionSettings{ MaxModelToolSteps: 99, MaxRuntime: time.Hour, MaxConsecutiveGateFailures: 5, - }, 4) + }, 4); err != nil { + t.Fatalf("applyWorkflowExecutionSettings: %v", err) + } if err := b.reloadConfig(t.Context(), &configuration.Document{Config: fileConfig()}); err != nil { t.Fatalf("reloadConfig: %v", err)