feat(archied): check a live settings replacement before switching to it

docs/prds/runtime-control-plane.md ("API") requires a live change to start
and check the new component before switching to it, keeping the old one
running when the check fails. The live workflow-execution-settings apply
stored whatever arrived on the watch, republished the configuration snapshot
new tasks are built from, and reported success unconditionally.

Stage the update as a candidate first: validate the limits and build the
snapshot they produce without touching anything running, then switch both in
only if the check passes. A refused update leaves the running limits and the
snapshot in place and reports the failure through the apply-status surface,
which keeps the version that is still live and carries the error. Boot
propagates the same rejection, so a process is never started on limits it
cannot run.

personas is the other live-watched kind and gets no check or report here;
filed as archie-core-nw7t rather than half-wired.

Closes archie-core-nwa0.
This commit is contained in:
sam committed 2026-09-21 20:32:04 +10:00
1 parent bf2cb79dc1
commit 83974b6847
3 files changed
+223 -10

No files matched your search

+56 -8
View File
@@ -90,7 +90,9 @@ func (b *boot) startWorkflowExecutionSettings(ctx context.Context) error {
if err != nil {
return err
}
b.applyWorkflowExecutionSettings(ctx, settings, version)
if err := b.applyWorkflowExecutionSettings(ctx, settings, version); err != nil {
return fmt.Errorf("apply workflow execution settings: %w", err)
}
updates, err := b.controlPlane.WatchWorkflowExecutionSettings(ctx, version)
if err != nil {
return err
@@ -101,22 +103,68 @@ func (b *boot) startWorkflowExecutionSettings(ctx context.Context) error {
b.log.Error("workflow execution settings watch failed", "err", update.Err)
return
}
b.applyWorkflowExecutionSettings(ctx, update.Settings, update.Version)
// A refused update leaves the running component in place and is
// already reported through apply status, so the watch keeps
// streaming instead of stopping on it.
_ = b.applyWorkflowExecutionSettings(ctx, update.Settings, update.Version)
}
}()
return nil
}
// applyWorkflowExecutionSettings records the settings as well as applying
// them: they arrive on a watch rather than in the file document, so a reload
// has nowhere else to read them back from.
func (b *boot) applyWorkflowExecutionSettings(ctx context.Context, settings workflow.ExecutionSettings, version int64) {
b.executionSettings.Store(&settings)
// executionSettingsCandidate is a live workflow-execution-settings update that
// has been built and checked but not yet switched in: the limits themselves,
// and the configuration snapshot every new task would be built from.
type executionSettingsCandidate struct {
settings workflow.ExecutionSettings
cfg config.Config
}
// buildExecutionSettingsCandidate stages a live update without touching
// anything running: it checks the candidate the running limits would be
// replaced with, so a change this process cannot run is refused before it
// becomes the live one (docs/prds/runtime-control-plane.md, "API": Archie
// starts and checks the new one before switching).
//
// The check runs here rather than only where the document was written because
// a live update is promoted long after that: the store's validation describes
// what may be stored, and this is the process that has to run it.
func (b *boot) buildExecutionSettingsCandidate(settings workflow.ExecutionSettings) (executionSettingsCandidate, error) {
if err := settings.Validate(); err != nil {
return executionSettingsCandidate{}, fmt.Errorf("workflow execution settings: %w", err)
}
cfg := b.cfgHolder.Get().Clone()
applyExecutionBudgets(&cfg, settings)
b.cfgHolder.Set(cfg)
return executionSettingsCandidate{settings: settings, cfg: cfg}, nil
}
// applyWorkflowExecutionSettings builds a candidate and switches it in.
//
// Both halves of the running component move together and only after the check
// passes: on a refusal the previous settings keep running, the configuration
// snapshot new tasks are built from is left as it is, and the failure is
// reported against the version that is still live, so the settings page shows
// a rejected edit rather than a version this process never ran.
//
// The settings are recorded as well as applied: they arrive on a watch rather
// than in the file document, so a reload has nowhere else to read them back
// from.
//
// The returned error is for a caller that cannot continue on limits that were
// not applied -- boot, which must not start a process it cannot run. On the
// watch path the running component stays live and the report is the outcome.
func (b *boot) applyWorkflowExecutionSettings(ctx context.Context, settings workflow.ExecutionSettings, version int64) error {
candidate, err := b.buildExecutionSettingsCandidate(settings)
if err != nil {
b.applyStatus.Report(ctx, controlplane.WorkflowExecutionSettingsKind, version, err)
b.log.Error("workflow execution settings rejected; the running settings stay", "version", version, "err", err)
return err
}
b.executionSettings.Store(&candidate.settings)
b.cfgHolder.Set(candidate.cfg)
b.applyStatus.Report(ctx, controlplane.WorkflowExecutionSettingsKind, version, nil)
b.log.Info("workflow execution settings applied", "version", version)
return nil
}
func applyExecutionBudgets(cfg *config.Config, settings workflow.ExecutionSettings) {
@@ -0,0 +1,163 @@
package archied
import (
"context"
"log/slog"
"sync"
"testing"
"time"
"github.com/samcharles93/archie-core/internal/app/controlplane"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/domain/applystatus"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
"github.com/samcharles93/archie-core/internal/domain/workflow"
)
// applyStatusRecorder is the State Store end of the apply-status surface: it
// keeps what a process reported, so a test can assert what the settings page
// would render.
type applyStatusRecorder struct {
mu sync.Mutex
writes []storecontract.ApplyStatus
}
func (r *applyStatusRecorder) PutApplyStatus(_ context.Context, status storecontract.ApplyStatus) error {
r.mu.Lock()
defer r.mu.Unlock()
r.writes = append(r.writes, status)
return nil
}
func (r *applyStatusRecorder) ListApplyStatus(context.Context) ([]storecontract.ApplyStatus, error) {
r.mu.Lock()
defer r.mu.Unlock()
return append([]storecontract.ApplyStatus(nil), r.writes...), nil
}
// last returns the most recent record written for kind.
func (r *applyStatusRecorder) last(kind string) (storecontract.ApplyStatus, bool) {
r.mu.Lock()
defer r.mu.Unlock()
for i := len(r.writes) - 1; i >= 0; i-- {
if r.writes[i].Kind == kind {
return r.writes[i], true
}
}
return storecontract.ApplyStatus{}, false
}
// newLiveApplyBoot builds a boot whose live kinds report apply status through
// a recorder, so a test can read back what the settings page would show.
func newLiveApplyBoot(t *testing.T) (*boot, *applyStatusRecorder) {
t.Helper()
recorder := &applyStatusRecorder{}
b := &boot{
cfg: fileConfig(),
log: slog.New(slog.DiscardHandler),
cfgHolder: config.NewHolder(fileConfig()),
processName: applystatus.Daemon,
}
b.applyStatus = applystatus.New(b.processName, recorder, b.log)
return b, recorder
}
// TestLiveExecutionSettingsUpdateIsCheckedBeforeItReplacesTheRunningLimits is
// archie-core-nwa0. docs/prds/runtime-control-plane.md ("API") requires a live
// change to replace a running component only after the new one has been
// started and checked; if the check fails the old one keeps running. A live
// workflow-execution-settings update replaces the limits every new task is
// built from, so a candidate the component refuses must leave the running
// limits and the running configuration snapshot in place, and the rejection
// must reach the apply-status surface carrying the version that is still live.
func TestLiveExecutionSettingsUpdateIsCheckedBeforeItReplacesTheRunningLimits(t *testing.T) {
// The limits the daemon is already running when the update arrives.
running := workflow.ExecutionSettings{MaxModelToolSteps: 10, MaxRuntime: 5 * time.Minute, MaxConsecutiveGateFailures: 3}
tests := []struct {
name string
candidate workflow.ExecutionSettings
wantRunning workflow.ExecutionSettings
wantError bool
wantVersion int64
}{
{
name: "an update the component accepts replaces the running limits",
candidate: workflow.ExecutionSettings{MaxModelToolSteps: 25, MaxRuntime: time.Hour, MaxConsecutiveGateFailures: 4},
wantRunning: workflow.ExecutionSettings{MaxModelToolSteps: 25, MaxRuntime: time.Hour, MaxConsecutiveGateFailures: 4},
wantVersion: 2,
},
{
// Zero is the documented "this individual limit is off", so an
// all-zero update is a legal live change, not a rejection.
name: "an update that disables every limit is accepted",
candidate: workflow.ExecutionSettings{},
wantRunning: workflow.ExecutionSettings{},
wantVersion: 2,
},
{
name: "an update with a negative step limit is refused and the running limits stay",
candidate: workflow.ExecutionSettings{MaxModelToolSteps: -1, MaxRuntime: time.Hour, MaxConsecutiveGateFailures: 4},
wantRunning: running,
wantError: true,
wantVersion: 1,
},
{
name: "an update with a negative runtime is refused and the running limits stay",
candidate: workflow.ExecutionSettings{MaxModelToolSteps: 25, MaxRuntime: -time.Second, MaxConsecutiveGateFailures: 4},
wantRunning: running,
wantError: true,
wantVersion: 1,
},
{
name: "an update with a negative gate-failure limit is refused and the running limits stay",
candidate: workflow.ExecutionSettings{MaxModelToolSteps: 25, MaxRuntime: time.Hour, MaxConsecutiveGateFailures: -1},
wantRunning: running,
wantError: true,
wantVersion: 1,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
b, status := newLiveApplyBoot(t)
if err := b.applyWorkflowExecutionSettings(t.Context(), running, 1); err != nil {
t.Fatalf("applying the running settings: %v", err)
}
if err := b.applyWorkflowExecutionSettings(t.Context(), tt.candidate, 2); (err != nil) != tt.wantError {
t.Fatalf("live update error = %v, want an error: %v", err, tt.wantError)
}
got := b.executionSettings.Load()
if got == nil {
t.Fatal("no settings are recorded as running")
}
if *got != tt.wantRunning {
t.Errorf("running settings = %+v, want %+v", *got, tt.wantRunning)
}
// The snapshot a new task is built from must move with the
// settings, and must not move for a refused update.
wantBudgets := config.Budgets{
MaxSteps: tt.wantRunning.MaxModelToolSteps,
WallClock: config.Duration(tt.wantRunning.MaxRuntime),
GateMaxFailures: tt.wantRunning.MaxConsecutiveGateFailures,
}
if budgets := b.cfgHolder.Get().Budgets; budgets != wantBudgets {
t.Errorf("running budgets = %+v, want %+v", budgets, wantBudgets)
}
last, ok := status.last(controlplane.WorkflowExecutionSettingsKind)
if !ok {
t.Fatal("the live update reported no apply status")
}
if last.AppliedVersion != tt.wantVersion {
t.Errorf("apply status version = %d, want %d", last.AppliedVersion, tt.wantVersion)
}
if (last.Error != "") != tt.wantError {
t.Errorf("apply status error = %q, want an error: %v", last.Error, tt.wantError)
}
})
}
}
@@ -102,9 +102,11 @@ func TestReloadConfigKeepsDatabaseOwnedSettings(t *testing.T) {
if err := b.loadRuntimeConfig(t.Context()); err != nil {
t.Fatalf("loadRuntimeConfig: %v", err)
}
b.applyWorkflowExecutionSettings(t.Context(), workflow.ExecutionSettings{
if err := b.applyWorkflowExecutionSettings(t.Context(), workflow.ExecutionSettings{
MaxModelToolSteps: 99, MaxRuntime: time.Hour, MaxConsecutiveGateFailures: 5,
}, 4)
}, 4); err != nil {
t.Fatalf("applyWorkflowExecutionSettings: %v", err)
}
if err := b.reloadConfig(t.Context(), &configuration.Document{Config: fileConfig()}); err != nil {
t.Fatalf("reloadConfig: %v", err)