feat(archied): check a live settings replacement before switching to it

Resolves archie-core-nwa0. The live apply path builds and probes a replacement
before promoting it and retains the previous component when the probe fails,
reporting the outcome through the apply-status surface instead of a hard-coded
nil.
This commit is contained in:
sam committed 2026-09-21 20:33:41 +10:00
commit 8a8cca5654
3 files changed
+435 -11

No files matched your search

+73 -8
View File
@@ -90,7 +90,9 @@ func (b *boot) startWorkflowExecutionSettings(ctx context.Context) error {
if err != nil {
return err
}
b.applyWorkflowExecutionSettings(ctx, settings, version)
if err := b.applyWorkflowExecutionSettings(ctx, settings, version); err != nil {
return fmt.Errorf("apply workflow execution settings: %w", err)
}
updates, err := b.controlPlane.WatchWorkflowExecutionSettings(ctx, version)
if err != nil {
return err
@@ -98,25 +100,88 @@ func (b *boot) startWorkflowExecutionSettings(ctx context.Context) error {
go func() {
for update := range updates {
if update.Err != nil {
// A refused update arrives here, not through
// applyWorkflowExecutionSettings: controlplane.Client decodes every
// document it is handed, so a stored value this process cannot run
// never reaches the apply call. Report it against the version it
// came from -- the reporter keeps the version it last applied --
// so the settings page shows the refusal instead of a version this
// process never ran (docs/prds/control-plane-apply-status.md).
//
// The stream is not re-established after this point; that gap is
// archie-core-yrmr, not this apply path.
b.applyStatus.Report(ctx, controlplane.WorkflowExecutionSettingsKind, update.Version, update.Err)
b.log.Error("workflow execution settings watch failed", "err", update.Err)
return
}
b.applyWorkflowExecutionSettings(ctx, update.Settings, update.Version)
_ = b.applyWorkflowExecutionSettings(ctx, update.Settings, update.Version)
}
}()
return nil
}
// applyWorkflowExecutionSettings records the settings as well as applying
// them: they arrive on a watch rather than in the file document, so a reload
// has nowhere else to read them back from.
func (b *boot) applyWorkflowExecutionSettings(ctx context.Context, settings workflow.ExecutionSettings, version int64) {
b.executionSettings.Store(&settings)
// executionSettingsCandidate is a live workflow-execution-settings update that
// has been built and checked but not yet switched in: the limits themselves,
// and the configuration snapshot every new task would be built from.
type executionSettingsCandidate struct {
settings workflow.ExecutionSettings
cfg config.Config
}
// buildExecutionSettingsCandidate stages a live update without touching
// anything running: it builds the configuration snapshot the new limits would
// be published as and runs this process's own runnability check on it -- the
// same configuration.Validate that runtimeConfig applies before publishing a
// layered config, and that the readiness probe applies to the published one. A
// snapshot this process cannot run is refused before it becomes the live one
// (docs/prds/runtime-control-plane.md, "API": Archie starts and checks the new
// one before switching).
//
// The kind's schema is deliberately not re-checked here. The store validates
// every document before it is written (Definition.Decode) and
// controlplane.Client validates it again as it decodes it, so re-running
// workflow.ExecutionSettings.Validate at this point would be a guard no
// producer can trip.
func (b *boot) buildExecutionSettingsCandidate(settings workflow.ExecutionSettings) (executionSettingsCandidate, error) {
cfg := b.cfgHolder.Get().Clone()
applyExecutionBudgets(&cfg, settings)
b.cfgHolder.Set(cfg)
if err := configuration.Validate(&cfg); err != nil {
return executionSettingsCandidate{}, fmt.Errorf("workflow execution settings: %w", err)
}
return executionSettingsCandidate{settings: settings, cfg: cfg}, nil
}
// applyWorkflowExecutionSettings builds a candidate and switches it in.
//
// Both halves of the running component move together and only after the check
// passes: on a refusal the previous settings keep running, the configuration
// snapshot new tasks are built from is left as it is, and the failure is
// reported for the version that was refused. The apply-status reporter writes
// that record against the version it last applied, so once this process has
// applied anything the settings page shows a rejected edit rather than a
// version this process never ran.
//
// The settings are recorded as well as applied: they arrive on a watch rather
// than in the file document, so a reload has nowhere else to read them back
// from.
//
// The returned error is propagated by boot, the caller that cannot continue on
// limits it did not apply. Boot reaches this function only with settings the
// control plane already decoded and over a snapshot loadRuntimeConfig already
// validated, so that propagation is the fail-closed direction of the rule
// above rather than a branch the daemon takes today.
func (b *boot) applyWorkflowExecutionSettings(ctx context.Context, settings workflow.ExecutionSettings, version int64) error {
candidate, err := b.buildExecutionSettingsCandidate(settings)
if err != nil {
b.applyStatus.Report(ctx, controlplane.WorkflowExecutionSettingsKind, version, err)
b.log.Error("workflow execution settings rejected; the running settings stay", "version", version, "err", err)
return err
}
b.executionSettings.Store(&candidate.settings)
b.cfgHolder.Set(candidate.cfg)
b.applyStatus.Report(ctx, controlplane.WorkflowExecutionSettingsKind, version, nil)
b.log.Info("workflow execution settings applied", "version", version)
return nil
}
func applyExecutionBudgets(cfg *config.Config, settings workflow.ExecutionSettings) {
@@ -0,0 +1,354 @@
package archied
import (
"context"
"io"
"log/slog"
"slices"
"strings"
"sync"
"testing"
"time"
"google.golang.org/grpc"
"github.com/samcharles93/archie-core/internal/app/controlplane"
"github.com/samcharles93/archie-core/internal/config"
pb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1"
"github.com/samcharles93/archie-core/internal/domain/applystatus"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
"github.com/samcharles93/archie-core/internal/domain/workflow"
)
// The limits this package's fixtures boot or run with, and the documents the
// control plane stores them as. Named so a test that asserts "the running
// limits stayed" and a test that asserts "the update was applied" cannot
// disagree about what "running" means.
var (
bootedSettings = workflow.ExecutionSettings{MaxModelToolSteps: 10, MaxRuntime: 5 * time.Minute, MaxConsecutiveGateFailures: 3}
bootedDocument = `{"max_model_tool_steps": 10, "max_runtime_seconds": 300, "max_consecutive_gate_failures": 3}`
)
// applyStatusRecorder is the State Store end of the apply-status surface: it
// keeps what a process reported, so a test can assert what the settings page
// would render.
type applyStatusRecorder struct {
mu sync.Mutex
writes []storecontract.ApplyStatus
}
func (r *applyStatusRecorder) PutApplyStatus(_ context.Context, status storecontract.ApplyStatus) error {
r.mu.Lock()
defer r.mu.Unlock()
r.writes = append(r.writes, status)
return nil
}
func (r *applyStatusRecorder) ListApplyStatus(context.Context) ([]storecontract.ApplyStatus, error) {
r.mu.Lock()
defer r.mu.Unlock()
return append([]storecontract.ApplyStatus(nil), r.writes...), nil
}
// last returns the most recent record written for kind.
func (r *applyStatusRecorder) last(kind string) (storecontract.ApplyStatus, bool) {
r.mu.Lock()
defer r.mu.Unlock()
for _, v := range slices.Backward(r.writes) {
if v.Kind == kind {
return v, true
}
}
return storecontract.ApplyStatus{}, false
}
// awaitCount waits until n records have been written for kind. The live apply
// runs on the watch goroutine, so a test that drives it through the stream has
// nothing to synchronise on but the record it is waiting for.
func (r *applyStatusRecorder) awaitCount(t *testing.T, kind string, n int) []storecontract.ApplyStatus {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
for {
r.mu.Lock()
written := make([]storecontract.ApplyStatus, 0, n)
for _, status := range r.writes {
if status.Kind == kind {
written = append(written, status)
}
}
r.mu.Unlock()
if len(written) >= n {
return written
}
if time.Now().After(deadline) {
t.Fatalf("%s: %d records reported, want %d", kind, len(written), n)
}
time.Sleep(5 * time.Millisecond)
}
}
// newLiveApplyBoot builds a boot running the given snapshot, whose apply status
// reports land in a recorder.
func newLiveApplyBoot(t *testing.T, running config.Config) (*boot, *applyStatusRecorder) {
t.Helper()
recorder := &applyStatusRecorder{}
b := &boot{
cfg: running,
log: slog.New(slog.DiscardHandler),
cfgHolder: config.NewHolder(running),
processName: applystatus.Daemon,
}
b.applyStatus = applystatus.New(b.processName, recorder, b.log)
return b, recorder
}
// settingsWatchStub answers Query with one stored settings document and Watch
// with a stream carrying the given documents. It is the seam the daemon really
// meets a live update through: the control plane stores documents, and
// controlplane.Client decodes them before the apply path sees anything.
type settingsWatchStub struct {
stored string
version int64
documents []string
}
func (*settingsWatchStub) Catalog(context.Context, *pb.CatalogRequest, ...grpc.CallOption) (*pb.CatalogResponse, error) {
return &pb.CatalogResponse{}, nil
}
func (s *settingsWatchStub) Query(context.Context, *pb.QueryRequest, ...grpc.CallOption) (*pb.QueryResponse, error) {
return &pb.QueryResponse{Resource: &pb.Resource{Kind: controlplane.WorkflowExecutionSettingsKind, Version: s.version, ValueJson: []byte(s.stored)}}, nil
}
func (*settingsWatchStub) History(context.Context, *pb.HistoryRequest, ...grpc.CallOption) (*pb.HistoryResponse, error) {
panic("unexpected History")
}
func (*settingsWatchStub) Command(context.Context, *pb.CommandRequest, ...grpc.CallOption) (*pb.CommandResponse, error) {
panic("unexpected Command")
}
func (s *settingsWatchStub) Watch(context.Context, *pb.WatchRequest, ...grpc.CallOption) (grpc.ServerStreamingClient[pb.WatchResponse], error) {
responses := make([]*pb.WatchResponse, 0, len(s.documents))
for i, document := range s.documents {
responses = append(responses, &pb.WatchResponse{Resource: &pb.Resource{
Kind: controlplane.WorkflowExecutionSettingsKind,
Version: s.version + int64(i) + 1,
ValueJson: []byte(document),
}})
}
return &watchStreamStub{responses: responses}, nil
}
// watchStreamStub is the client end of a Watch stream. Recv is the only method
// the settings watch calls; the embedded ClientStream is nil, so any other call
// panics rather than pretending to work.
type watchStreamStub struct {
grpc.ClientStream
responses []*pb.WatchResponse
next int
}
func (s *watchStreamStub) Recv() (*pb.WatchResponse, error) {
if s.next >= len(s.responses) {
return nil, io.EOF
}
response := s.responses[s.next]
s.next++
return response, nil
}
// TestLiveExecutionSettingsUpdateIsCheckedBeforeItReplacesTheRunningLimits is
// archie-core-nwa0. docs/prds/runtime-control-plane.md ("API") requires a live
// change to replace a running component only after the new one has been
// started and checked; if the check fails the old one keeps running. A live
// workflow-execution-settings update replaces the configuration snapshot every
// new task is built from, so a candidate whose snapshot this process cannot run
// must leave the running limits and the running snapshot in place, and the
// failure must reach the apply-status surface carrying the version that is
// still live.
func TestLiveExecutionSettingsUpdateIsCheckedBeforeItReplacesTheRunningLimits(t *testing.T) {
tests := []struct {
name string
candidate workflow.ExecutionSettings
wantRunning workflow.ExecutionSettings
wantVersion int64
}{
{
name: "an update whose snapshot this process can run replaces the running limits",
candidate: workflow.ExecutionSettings{MaxModelToolSteps: 25, MaxRuntime: time.Hour, MaxConsecutiveGateFailures: 4},
wantRunning: workflow.ExecutionSettings{MaxModelToolSteps: 25, MaxRuntime: time.Hour, MaxConsecutiveGateFailures: 4},
wantVersion: 2,
},
{
// Zero is the documented "this individual limit is off", so an
// all-zero update is a legal live change, not a rejection, and the
// check must not invent a bound the store does not enforce.
name: "an update that disables every limit is accepted",
candidate: workflow.ExecutionSettings{},
wantRunning: workflow.ExecutionSettings{},
wantVersion: 2,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
b, status := newLiveApplyBoot(t, fileConfig())
if err := b.applyWorkflowExecutionSettings(t.Context(), bootedSettings, 1); err != nil {
t.Fatalf("applying the running settings: %v", err)
}
if err := b.applyWorkflowExecutionSettings(t.Context(), tt.candidate, 2); err != nil {
t.Fatalf("live update: %v", err)
}
got := b.executionSettings.Load()
if got == nil {
t.Fatal("no settings are recorded as running")
}
if *got != tt.wantRunning {
t.Errorf("running settings = %+v, want %+v", *got, tt.wantRunning)
}
// The snapshot a new task is built from has to move with the
// settings: it is the half of the component the daemon runs.
wantBudgets := budgetsFor(tt.wantRunning)
if budgets := b.cfgHolder.Get().Budgets; budgets != wantBudgets {
t.Errorf("running budgets = %+v, want %+v", budgets, wantBudgets)
}
last, ok := status.last(controlplane.WorkflowExecutionSettingsKind)
if !ok {
t.Fatal("the live update reported no apply status")
}
if last.AppliedVersion != tt.wantVersion {
t.Errorf("apply status version = %d, want %d", last.AppliedVersion, tt.wantVersion)
}
if last.Error != "" {
t.Errorf("apply status error = %q, want none for an applied update", last.Error)
}
})
}
}
// TestLiveExecutionSettingsUpdateIsRefusedWhenTheSnapshotIsNotRunnable covers
// the refusal half of the same check. The candidate is the running snapshot
// with new limits written into it, and configuration.Validate does not read
// limits, so the only input that can fail the check is a snapshot this process
// would not run -- which is what this test's daemon is running. A live update
// must not publish a snapshot this process cannot run, must leave the component
// it would have replaced in place, and must report the refusal against the
// version still live.
func TestLiveExecutionSettingsUpdateIsRefusedWhenTheSnapshotIsNotRunnable(t *testing.T) {
unrunnable := fileConfig()
unrunnable.Containers.Image = ""
applyExecutionBudgets(&unrunnable, bootedSettings)
b, status := newLiveApplyBoot(t, unrunnable)
// The daemon is already running these limits. Seeded directly rather than
// applied: the ordinary apply refuses this snapshot too, which is the point
// of the check.
b.executionSettings.Store(&bootedSettings)
b.applyStatus.Report(t.Context(), controlplane.WorkflowExecutionSettingsKind, 1, nil)
budgets := budgetsFor(bootedSettings)
update := workflow.ExecutionSettings{MaxModelToolSteps: 25, MaxRuntime: time.Hour, MaxConsecutiveGateFailures: 4}
if err := b.applyWorkflowExecutionSettings(t.Context(), update, 2); err == nil {
t.Fatal("an update whose snapshot this process cannot run was applied")
}
if got := b.executionSettings.Load(); got == nil || *got != bootedSettings {
t.Errorf("running settings = %v, want the refused update to leave %+v running", got, bootedSettings)
}
if got := b.cfgHolder.Get().Budgets; got != budgets {
t.Errorf("running budgets = %+v, want the refused update to leave %+v running", got, budgets)
}
last, ok := status.last(controlplane.WorkflowExecutionSettingsKind)
if !ok {
t.Fatal("the refused update reported no apply status")
}
if last.Error == "" {
t.Error("the refusal was not reported through apply status")
}
if last.AppliedVersion != 1 {
t.Errorf("apply status version = %d, want the version still live (1)", last.AppliedVersion)
}
}
// TestLiveExecutionSettingsRefusalIsReportedThroughTheClient is the other half
// of the same item, over the seam the daemon actually meets a refusal through.
// controlplane.Client decodes every document it is handed, so an update this
// process cannot run arrives as a watch error, never as invalid settings: the
// watch branch must report that refusal against the version still live instead
// of only logging it, or the settings page shows a live resource as healthy
// while nothing has been applied. (That the stream is not re-established
// afterwards is archie-core-yrmr, not this apply path.)
func TestLiveExecutionSettingsRefusalIsReportedThroughTheClient(t *testing.T) {
tests := []struct {
name string
document string
wantRunning workflow.ExecutionSettings
wantVersion int64
wantErrorContains string
}{
{
name: "an update the process can run is applied",
document: `{"max_model_tool_steps": 25, "max_runtime_seconds": 3600, "max_consecutive_gate_failures": 4}`,
wantRunning: workflow.ExecutionSettings{MaxModelToolSteps: 25, MaxRuntime: time.Hour, MaxConsecutiveGateFailures: 4},
wantVersion: 2,
},
{
name: "an update the client refuses is reported against the version still live",
document: `{"max_model_tool_steps": -1, "max_runtime_seconds": 3600, "max_consecutive_gate_failures": 4}`,
wantRunning: bootedSettings,
wantVersion: 1,
wantErrorContains: "max model/tool steps must not be negative",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
b, status := newLiveApplyBoot(t, fileConfig())
b.controlPlane = controlplane.NewRPCClient(&settingsWatchStub{stored: bootedDocument, version: 1, documents: []string{tt.document}})
if err := b.startWorkflowExecutionSettings(t.Context()); err != nil {
t.Fatalf("startWorkflowExecutionSettings: %v", err)
}
// One record for boot, one for the watched update.
records := status.awaitCount(t, controlplane.WorkflowExecutionSettingsKind, 2)
got := b.executionSettings.Load()
if got == nil {
t.Fatal("no settings are recorded as running")
}
if *got != tt.wantRunning {
t.Errorf("running settings = %+v, want %+v", *got, tt.wantRunning)
}
if budgets := b.cfgHolder.Get().Budgets; budgets != budgetsFor(tt.wantRunning) {
t.Errorf("running budgets = %+v, want %+v", budgets, budgetsFor(tt.wantRunning))
}
last := records[len(records)-1]
if last.AppliedVersion != tt.wantVersion {
t.Errorf("apply status version = %d, want %d", last.AppliedVersion, tt.wantVersion)
}
if tt.wantErrorContains == "" && last.Error != "" {
t.Errorf("apply status error = %q, want none for an applied update", last.Error)
}
if tt.wantErrorContains != "" && !strings.Contains(last.Error, tt.wantErrorContains) {
t.Errorf("apply status error = %q, want it to carry %q", last.Error, tt.wantErrorContains)
}
})
}
}
// budgetsFor is the configuration snapshot a set of limits produces. Named so
// the tests and applyExecutionBudgets cannot drift on the mapping.
func budgetsFor(settings workflow.ExecutionSettings) config.Budgets {
return config.Budgets{
MaxSteps: settings.MaxModelToolSteps,
WallClock: config.Duration(settings.MaxRuntime),
GateMaxFailures: settings.MaxConsecutiveGateFailures,
}
}
@@ -65,7 +65,9 @@ func databaseOwnedResources() map[string]any {
}
// fileConfig is what the loader produces from config.toml alone: the layer a
// SIGHUP reload re-resolves, holding none of the database's values.
// SIGHUP reload re-resolves, holding none of the database's values. It is a
// config the daemon would run, so it passes configuration.Validate -- the live
// apply path runs that same check over the snapshot it is about to publish.
func fileConfig() config.Config {
return config.Config{
BotUser: "widget",
@@ -79,6 +81,7 @@ func fileConfig() config.Config {
PluginDir: "/file/plugins",
Containers: config.ContainerConfig{Image: "archie:from-file", PullPolicy: "missing"},
Budgets: config.Budgets{MaxSteps: 1, WallClock: config.Duration(time.Minute)},
Dispatch: config.Dispatch{Trigger: "assignee"},
}
}
@@ -102,9 +105,11 @@ func TestReloadConfigKeepsDatabaseOwnedSettings(t *testing.T) {
if err := b.loadRuntimeConfig(t.Context()); err != nil {
t.Fatalf("loadRuntimeConfig: %v", err)
}
b.applyWorkflowExecutionSettings(t.Context(), workflow.ExecutionSettings{
if err := b.applyWorkflowExecutionSettings(t.Context(), workflow.ExecutionSettings{
MaxModelToolSteps: 99, MaxRuntime: time.Hour, MaxConsecutiveGateFailures: 5,
}, 4)
}, 4); err != nil {
t.Fatalf("applyWorkflowExecutionSettings: %v", err)
}
if err := b.reloadConfig(t.Context(), &configuration.Document{Config: fileConfig()}); err != nil {
t.Fatalf("reloadConfig: %v", err)