feat(archied): check a live settings replacement before switching to it
docs/prds/runtime-control-plane.md ("API") requires a live change to start
and check the new component before switching to it, keeping the old one
running when the check fails. The live workflow-execution-settings apply
stored whatever arrived on the watch, republished the configuration snapshot
new tasks are built from, and reported success unconditionally.
Stage the update as a candidate first: validate the limits and build the
snapshot they produce without touching anything running, then switch both in
only if the check passes. A refused update leaves the running limits and the
snapshot in place and reports the failure through the apply-status surface,
which keeps the version that is still live and carries the error. Boot
propagates the same rejection, so a process is never started on limits it
cannot run.
personas is the other live-watched kind and gets no check or report here;
filed as archie-core-nw7t rather than half-wired.
Closes archie-core-nwa0.
This commit is contained in:
1 parent
bf2cb79dc1
commit
83974b6847
3 files changed
+223
-10
No files matched your search
@@ -90,7 +90,9 @@ func (b *boot) startWorkflowExecutionSettings(ctx context.Context) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b.applyWorkflowExecutionSettings(ctx, settings, version)
|
||||
if err := b.applyWorkflowExecutionSettings(ctx, settings, version); err != nil {
|
||||
return fmt.Errorf("apply workflow execution settings: %w", err)
|
||||
}
|
||||
updates, err := b.controlPlane.WatchWorkflowExecutionSettings(ctx, version)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -101,22 +103,68 @@ func (b *boot) startWorkflowExecutionSettings(ctx context.Context) error {
|
||||
b.log.Error("workflow execution settings watch failed", "err", update.Err)
|
||||
return
|
||||
}
|
||||
b.applyWorkflowExecutionSettings(ctx, update.Settings, update.Version)
|
||||
// A refused update leaves the running component in place and is
|
||||
// already reported through apply status, so the watch keeps
|
||||
// streaming instead of stopping on it.
|
||||
_ = b.applyWorkflowExecutionSettings(ctx, update.Settings, update.Version)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
// applyWorkflowExecutionSettings records the settings as well as applying
|
||||
// them: they arrive on a watch rather than in the file document, so a reload
|
||||
// has nowhere else to read them back from.
|
||||
func (b *boot) applyWorkflowExecutionSettings(ctx context.Context, settings workflow.ExecutionSettings, version int64) {
|
||||
b.executionSettings.Store(&settings)
|
||||
// executionSettingsCandidate is a live workflow-execution-settings update that
|
||||
// has been built and checked but not yet switched in: the limits themselves,
|
||||
// and the configuration snapshot every new task would be built from.
|
||||
type executionSettingsCandidate struct {
|
||||
settings workflow.ExecutionSettings
|
||||
cfg config.Config
|
||||
}
|
||||
|
||||
// buildExecutionSettingsCandidate stages a live update without touching
|
||||
// anything running: it checks the candidate the running limits would be
|
||||
// replaced with, so a change this process cannot run is refused before it
|
||||
// becomes the live one (docs/prds/runtime-control-plane.md, "API": Archie
|
||||
// starts and checks the new one before switching).
|
||||
//
|
||||
// The check runs here rather than only where the document was written because
|
||||
// a live update is promoted long after that: the store's validation describes
|
||||
// what may be stored, and this is the process that has to run it.
|
||||
func (b *boot) buildExecutionSettingsCandidate(settings workflow.ExecutionSettings) (executionSettingsCandidate, error) {
|
||||
if err := settings.Validate(); err != nil {
|
||||
return executionSettingsCandidate{}, fmt.Errorf("workflow execution settings: %w", err)
|
||||
}
|
||||
cfg := b.cfgHolder.Get().Clone()
|
||||
applyExecutionBudgets(&cfg, settings)
|
||||
b.cfgHolder.Set(cfg)
|
||||
return executionSettingsCandidate{settings: settings, cfg: cfg}, nil
|
||||
}
|
||||
|
||||
// applyWorkflowExecutionSettings builds a candidate and switches it in.
|
||||
//
|
||||
// Both halves of the running component move together and only after the check
|
||||
// passes: on a refusal the previous settings keep running, the configuration
|
||||
// snapshot new tasks are built from is left as it is, and the failure is
|
||||
// reported against the version that is still live, so the settings page shows
|
||||
// a rejected edit rather than a version this process never ran.
|
||||
//
|
||||
// The settings are recorded as well as applied: they arrive on a watch rather
|
||||
// than in the file document, so a reload has nowhere else to read them back
|
||||
// from.
|
||||
//
|
||||
// The returned error is for a caller that cannot continue on limits that were
|
||||
// not applied -- boot, which must not start a process it cannot run. On the
|
||||
// watch path the running component stays live and the report is the outcome.
|
||||
func (b *boot) applyWorkflowExecutionSettings(ctx context.Context, settings workflow.ExecutionSettings, version int64) error {
|
||||
candidate, err := b.buildExecutionSettingsCandidate(settings)
|
||||
if err != nil {
|
||||
b.applyStatus.Report(ctx, controlplane.WorkflowExecutionSettingsKind, version, err)
|
||||
b.log.Error("workflow execution settings rejected; the running settings stay", "version", version, "err", err)
|
||||
return err
|
||||
}
|
||||
b.executionSettings.Store(&candidate.settings)
|
||||
b.cfgHolder.Set(candidate.cfg)
|
||||
b.applyStatus.Report(ctx, controlplane.WorkflowExecutionSettingsKind, version, nil)
|
||||
b.log.Info("workflow execution settings applied", "version", version)
|
||||
return nil
|
||||
}
|
||||
|
||||
func applyExecutionBudgets(cfg *config.Config, settings workflow.ExecutionSettings) {
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
package archied
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/app/controlplane"
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/domain/applystatus"
|
||||
"github.com/samcharles93/archie-core/internal/domain/storecontract"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
||||
)
|
||||
|
||||
// applyStatusRecorder is the State Store end of the apply-status surface: it
|
||||
// keeps what a process reported, so a test can assert what the settings page
|
||||
// would render.
|
||||
type applyStatusRecorder struct {
|
||||
mu sync.Mutex
|
||||
writes []storecontract.ApplyStatus
|
||||
}
|
||||
|
||||
func (r *applyStatusRecorder) PutApplyStatus(_ context.Context, status storecontract.ApplyStatus) error {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.writes = append(r.writes, status)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *applyStatusRecorder) ListApplyStatus(context.Context) ([]storecontract.ApplyStatus, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return append([]storecontract.ApplyStatus(nil), r.writes...), nil
|
||||
}
|
||||
|
||||
// last returns the most recent record written for kind.
|
||||
func (r *applyStatusRecorder) last(kind string) (storecontract.ApplyStatus, bool) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
for i := len(r.writes) - 1; i >= 0; i-- {
|
||||
if r.writes[i].Kind == kind {
|
||||
return r.writes[i], true
|
||||
}
|
||||
}
|
||||
return storecontract.ApplyStatus{}, false
|
||||
}
|
||||
|
||||
// newLiveApplyBoot builds a boot whose live kinds report apply status through
|
||||
// a recorder, so a test can read back what the settings page would show.
|
||||
func newLiveApplyBoot(t *testing.T) (*boot, *applyStatusRecorder) {
|
||||
t.Helper()
|
||||
recorder := &applyStatusRecorder{}
|
||||
b := &boot{
|
||||
cfg: fileConfig(),
|
||||
log: slog.New(slog.DiscardHandler),
|
||||
cfgHolder: config.NewHolder(fileConfig()),
|
||||
processName: applystatus.Daemon,
|
||||
}
|
||||
b.applyStatus = applystatus.New(b.processName, recorder, b.log)
|
||||
return b, recorder
|
||||
}
|
||||
|
||||
// TestLiveExecutionSettingsUpdateIsCheckedBeforeItReplacesTheRunningLimits is
|
||||
// archie-core-nwa0. docs/prds/runtime-control-plane.md ("API") requires a live
|
||||
// change to replace a running component only after the new one has been
|
||||
// started and checked; if the check fails the old one keeps running. A live
|
||||
// workflow-execution-settings update replaces the limits every new task is
|
||||
// built from, so a candidate the component refuses must leave the running
|
||||
// limits and the running configuration snapshot in place, and the rejection
|
||||
// must reach the apply-status surface carrying the version that is still live.
|
||||
func TestLiveExecutionSettingsUpdateIsCheckedBeforeItReplacesTheRunningLimits(t *testing.T) {
|
||||
// The limits the daemon is already running when the update arrives.
|
||||
running := workflow.ExecutionSettings{MaxModelToolSteps: 10, MaxRuntime: 5 * time.Minute, MaxConsecutiveGateFailures: 3}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
candidate workflow.ExecutionSettings
|
||||
wantRunning workflow.ExecutionSettings
|
||||
wantError bool
|
||||
wantVersion int64
|
||||
}{
|
||||
{
|
||||
name: "an update the component accepts replaces the running limits",
|
||||
candidate: workflow.ExecutionSettings{MaxModelToolSteps: 25, MaxRuntime: time.Hour, MaxConsecutiveGateFailures: 4},
|
||||
wantRunning: workflow.ExecutionSettings{MaxModelToolSteps: 25, MaxRuntime: time.Hour, MaxConsecutiveGateFailures: 4},
|
||||
wantVersion: 2,
|
||||
},
|
||||
{
|
||||
// Zero is the documented "this individual limit is off", so an
|
||||
// all-zero update is a legal live change, not a rejection.
|
||||
name: "an update that disables every limit is accepted",
|
||||
candidate: workflow.ExecutionSettings{},
|
||||
wantRunning: workflow.ExecutionSettings{},
|
||||
wantVersion: 2,
|
||||
},
|
||||
{
|
||||
name: "an update with a negative step limit is refused and the running limits stay",
|
||||
candidate: workflow.ExecutionSettings{MaxModelToolSteps: -1, MaxRuntime: time.Hour, MaxConsecutiveGateFailures: 4},
|
||||
wantRunning: running,
|
||||
wantError: true,
|
||||
wantVersion: 1,
|
||||
},
|
||||
{
|
||||
name: "an update with a negative runtime is refused and the running limits stay",
|
||||
candidate: workflow.ExecutionSettings{MaxModelToolSteps: 25, MaxRuntime: -time.Second, MaxConsecutiveGateFailures: 4},
|
||||
wantRunning: running,
|
||||
wantError: true,
|
||||
wantVersion: 1,
|
||||
},
|
||||
{
|
||||
name: "an update with a negative gate-failure limit is refused and the running limits stay",
|
||||
candidate: workflow.ExecutionSettings{MaxModelToolSteps: 25, MaxRuntime: time.Hour, MaxConsecutiveGateFailures: -1},
|
||||
wantRunning: running,
|
||||
wantError: true,
|
||||
wantVersion: 1,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
b, status := newLiveApplyBoot(t)
|
||||
if err := b.applyWorkflowExecutionSettings(t.Context(), running, 1); err != nil {
|
||||
t.Fatalf("applying the running settings: %v", err)
|
||||
}
|
||||
if err := b.applyWorkflowExecutionSettings(t.Context(), tt.candidate, 2); (err != nil) != tt.wantError {
|
||||
t.Fatalf("live update error = %v, want an error: %v", err, tt.wantError)
|
||||
}
|
||||
|
||||
got := b.executionSettings.Load()
|
||||
if got == nil {
|
||||
t.Fatal("no settings are recorded as running")
|
||||
}
|
||||
if *got != tt.wantRunning {
|
||||
t.Errorf("running settings = %+v, want %+v", *got, tt.wantRunning)
|
||||
}
|
||||
|
||||
// The snapshot a new task is built from must move with the
|
||||
// settings, and must not move for a refused update.
|
||||
wantBudgets := config.Budgets{
|
||||
MaxSteps: tt.wantRunning.MaxModelToolSteps,
|
||||
WallClock: config.Duration(tt.wantRunning.MaxRuntime),
|
||||
GateMaxFailures: tt.wantRunning.MaxConsecutiveGateFailures,
|
||||
}
|
||||
if budgets := b.cfgHolder.Get().Budgets; budgets != wantBudgets {
|
||||
t.Errorf("running budgets = %+v, want %+v", budgets, wantBudgets)
|
||||
}
|
||||
|
||||
last, ok := status.last(controlplane.WorkflowExecutionSettingsKind)
|
||||
if !ok {
|
||||
t.Fatal("the live update reported no apply status")
|
||||
}
|
||||
if last.AppliedVersion != tt.wantVersion {
|
||||
t.Errorf("apply status version = %d, want %d", last.AppliedVersion, tt.wantVersion)
|
||||
}
|
||||
if (last.Error != "") != tt.wantError {
|
||||
t.Errorf("apply status error = %q, want an error: %v", last.Error, tt.wantError)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -102,9 +102,11 @@ func TestReloadConfigKeepsDatabaseOwnedSettings(t *testing.T) {
|
||||
if err := b.loadRuntimeConfig(t.Context()); err != nil {
|
||||
t.Fatalf("loadRuntimeConfig: %v", err)
|
||||
}
|
||||
b.applyWorkflowExecutionSettings(t.Context(), workflow.ExecutionSettings{
|
||||
if err := b.applyWorkflowExecutionSettings(t.Context(), workflow.ExecutionSettings{
|
||||
MaxModelToolSteps: 99, MaxRuntime: time.Hour, MaxConsecutiveGateFailures: 5,
|
||||
}, 4)
|
||||
}, 4); err != nil {
|
||||
t.Fatalf("applyWorkflowExecutionSettings: %v", err)
|
||||
}
|
||||
|
||||
if err := b.reloadConfig(t.Context(), &configuration.Document{Config: fileConfig()}); err != nil {
|
||||
t.Fatalf("reloadConfig: %v", err)
|
||||
|
||||
Reference in new issue
Block a user