Files
archie/internal/app/archied/control_plane.go
T
sam b8f5d0fc3d fix(archied): republish the config snapshot when live settings apply
applyWorkflowExecutionSettings swapped the running config with cfgHolder.Set,
so the dashboard's published projection kept the previous budgets until the
next reload. Route it through publishConfig like every other config change.

Fixes #1148
2026-09-27 05:09:24 +10:00

356 lines
16 KiB
Go

package archied
import (
"context"
"fmt"
"time"
"github.com/samcharles93/archie-core/internal/agentexec"
"github.com/samcharles93/archie-core/internal/app/controlplane"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/domain/workflow"
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
)
// runtimeConfig layers every database-owned setting over a file config and
// validates the result. Boot and the SIGHUP reload both go through it: the
// reload re-resolves the file document alone, so republishing that document
// as it stands reverts each of these layers to its file value until the
// process restarts (archie-core-ju85).
func (b *boot) runtimeConfig(ctx context.Context, base config.Config) (config.Config, map[string]int64, error) {
cfg, versions, err := b.controlPlane.RuntimeConfig(ctx, base)
if err != nil {
return config.Config{}, nil, err
}
if settings := b.executionSettings.Load(); settings != nil {
applyExecutionBudgets(&cfg, *settings)
}
if err := configuration.Validate(&cfg); err != nil {
wrapped := fmt.Errorf("validate database settings: %w", err)
b.reportApplied(ctx, versions, wrapped)
return config.Config{}, nil, wrapped
}
// The State Store persists source references, while openStores resolved
// only the file snapshot. Layering provider-settings restores the source
// references, so resolve the effective map before any runtime consumes it.
if err := resolveProviderSecrets(&cfg, b.secrets, b.log); err != nil {
b.reportApplied(ctx, versions, err)
return config.Config{}, nil, err
}
b.reportApplied(ctx, versions, nil)
return cfg, versions, nil
}
// reportApplied publishes the version of each kind this process just layered
// in. On a validation failure every kind is reported with the error, because
// the layering is all-or-nothing: none of the versions read took effect.
func (b *boot) reportApplied(ctx context.Context, versions map[string]int64, applyErr error) {
for kind, version := range versions {
b.applyStatus.Report(ctx, kind, version, applyErr)
}
}
func (b *boot) loadRuntimeConfig(ctx context.Context) error {
cfg, versions, err := b.runtimeConfig(ctx, b.cfg)
if err != nil {
return err
}
b.cfg = cfg
b.cfgHolder.Set(cfg)
// The versions boot layered in are the resume points the runtime-resource
// watches start from: a watch re-established after a version it already
// applied neither replays a document this process handled nor skips one
// it has not.
b.runtimeVersions = versions
return nil
}
// reloadConfig publishes a freshly resolved file document as the running
// config. The database layer is re-applied first, and a control plane that
// cannot answer aborts the reload: the running config already holds those
// values, so publishing the file's over them is the worse outcome.
func (b *boot) reloadConfig(ctx context.Context, doc *configuration.Document) error {
// Boot merges catalog-discovered providers into cfg before the
// Holders are seeded; publishing the raw reloaded document would
// drop them from the running config even though the file is
// unchanged. Re-apply the same merge (idempotent: a catalog that
// failed to load merges to identity).
applyModelCatalog(&doc.Config, b.catalog)
// Bounded: this runs on the signal loop, which handles nothing else
// while it waits, and a State Store that has stopped answering must
// surface as a failed reload rather than a SIGHUP that never returns.
queryCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
cfg, _, err := b.runtimeConfig(queryCtx, doc.Config)
if err != nil {
return fmt.Errorf("runtime settings unavailable: %w", err)
}
doc.Config = cfg
old := b.cfgHolder.Get()
b.currentProvenance.Store(&doc.Provenance)
b.publishConfig(ctx, cfg)
if fields := changedNonReloadableFields(old, cfg); len(fields) > 0 {
b.log.Warn("config reloaded; some changes require a restart",
"fields", fields, "paths", doc.Provenance.Paths())
} else {
b.log.Info("config reloaded", "paths", doc.Provenance.Paths())
}
return nil
}
// startWorkflowExecutionSettings loads the stored limits, applies them and
// keeps the stream that delivers later ones established. The first read and
// the first stream are synchronous, so a control plane that cannot be reached
// fails the boot that asked for it; after that the watch does not return, it
// reconnects (see keepWatch).
func (b *boot) startWorkflowExecutionSettings(ctx context.Context) error {
settings, version, err := b.controlPlane.WorkflowExecutionSettings(ctx)
if err != nil {
return err
}
if err := b.applyWorkflowExecutionSettings(ctx, settings, version); err != nil {
return fmt.Errorf("apply workflow execution settings: %w", err)
}
updates, err := b.controlPlane.WatchWorkflowExecutionSettings(ctx, version)
if err != nil {
return err
}
var lastApplyErr, streamErr error
open := func(ctx context.Context, after int64) (<-chan controlplane.AppliedSettings, error) {
updates, err := b.controlPlane.WatchWorkflowExecutionSettings(ctx, after)
if err == nil {
b.applyStatus.Report(ctx, controlplane.WorkflowExecutionSettingsKind, after, lastApplyErr)
}
return updates, err
}
go keepWatch(ctx, b.log, controlplane.WorkflowExecutionSettingsKind, version, updates,
open,
waitFor,
func(update controlplane.AppliedSettings) int64 { return update.Version },
func(update controlplane.AppliedSettings) {
if update.Err != nil {
// A refused document carries its version, while a stream failure
// does not. Keep a refusal visible across a transport outage; when
// there is no outstanding refusal, the end callback records the
// outage until a watch is re-established.
if update.Version > 0 {
lastApplyErr = update.Err
b.applyStatus.Report(ctx, controlplane.WorkflowExecutionSettingsKind, update.Version, update.Err)
} else {
streamErr = update.Err
}
b.log.Error("workflow execution settings watch failed", "err", update.Err)
return
}
lastApplyErr = b.applyWorkflowExecutionSettings(ctx, update.Settings, update.Version)
},
func() {
if lastApplyErr != nil {
return
}
if streamErr == nil {
streamErr = fmt.Errorf("control-plane watch stream ended")
}
b.applyStatus.Report(ctx, controlplane.WorkflowExecutionSettingsKind, 0,
fmt.Errorf("workflow execution settings watch unavailable: %w", streamErr))
streamErr = nil
})
return nil
}
// executionSettingsCandidate is a live workflow-execution-settings update that
// has been built and checked but not yet switched in: the limits themselves,
// and the configuration snapshot every new task would be built from.
type executionSettingsCandidate struct {
settings workflow.ExecutionSettings
cfg config.Config
}
// buildExecutionSettingsCandidate stages a live update without touching
// anything running: it builds the configuration snapshot the new limits would
// be published as and runs this process's own runnability check on it -- the
// same configuration.Validate that runtimeConfig applies before publishing a
// layered config, and that the readiness probe applies to the published one. A
// snapshot this process cannot run is refused before it becomes the live one
// (docs/prds/runtime-control-plane.md, "API": Archie starts and checks the new
// one before switching).
//
// The kind's schema is deliberately not re-checked here. The store validates
// every document before it is written (Definition.Decode) and
// controlplane.Client validates it again as it decodes it, so re-running
// workflow.ExecutionSettings.Validate at this point would be a guard no
// producer can trip.
func (b *boot) buildExecutionSettingsCandidate(settings workflow.ExecutionSettings) (executionSettingsCandidate, error) {
cfg := b.cfgHolder.Get().Clone()
applyExecutionBudgets(&cfg, settings)
if err := configuration.Validate(&cfg); err != nil {
return executionSettingsCandidate{}, fmt.Errorf("workflow execution settings: %w", err)
}
return executionSettingsCandidate{settings: settings, cfg: cfg}, nil
}
// applyWorkflowExecutionSettings builds a candidate and switches it in.
//
// Both halves of the running component move together and only after the check
// passes: on a refusal the previous settings keep running, the configuration
// snapshot new tasks are built from is left as it is, and the failure is
// reported for the version that was refused. The apply-status reporter writes
// that record against the version it last applied, so once this process has
// applied anything the settings page shows a rejected edit rather than a
// version this process never ran.
//
// The settings are recorded as well as applied: they arrive on a watch rather
// than in the file document, so a reload has nowhere else to read them back
// from.
//
// The returned error is propagated by boot, the caller that cannot continue on
// limits it did not apply. Boot reaches this function only with settings the
// control plane already decoded and over a snapshot loadRuntimeConfig already
// validated, so that propagation is the fail-closed direction of the rule
// above rather than a branch the daemon takes today.
func (b *boot) applyWorkflowExecutionSettings(ctx context.Context, settings workflow.ExecutionSettings, version int64) error {
candidate, err := b.buildExecutionSettingsCandidate(settings)
if err != nil {
b.applyStatus.Report(ctx, controlplane.WorkflowExecutionSettingsKind, version, err)
b.log.Error("workflow execution settings rejected; the running settings stay", "version", version, "err", err)
return err
}
b.executionSettings.Store(&candidate.settings)
b.publishConfig(ctx, candidate.cfg)
b.applyStatus.Report(ctx, controlplane.WorkflowExecutionSettingsKind, version, nil)
b.log.Info("workflow execution settings applied", "version", version)
if limit, uptime := settings.MaxTaskRuntime, candidate.cfg.Containers.MaxUptime.Std(); uptime > 0 && (limit <= 0 || limit > uptime) {
b.log.Warn("containers.max_uptime is shorter than the task time limit; the container is killed first", "max_uptime", uptime, "max_task_runtime", limit)
}
return nil
}
func applyExecutionBudgets(cfg *config.Config, settings workflow.ExecutionSettings) {
cfg.Budgets = config.Budgets{
MaxSteps: settings.MaxModelToolSteps, WallClock: config.Duration(settings.MaxRuntime), GateMaxFailures: settings.MaxConsecutiveGateFailures,
TaskWallClock: config.Duration(settings.MaxTaskRuntime),
}
for i := range cfg.Identities {
cfg.Identities[i].Budgets = cfg.Budgets
}
}
// startLiveSettings starts everything a live control-plane update needs
// established before Run continues: the workflow-execution-settings watch,
// whose limits the layering applies, and one watch per runtime-resource kind
// (control_plane.go, runtimeResourceKinds). Both first reads are synchronous,
// so a control plane that cannot be watched at all fails the boot that asked
// for it rather than leaving the process running settings it can no longer
// update.
func (b *boot) startLiveSettings(ctx context.Context) error {
if err := b.startWorkflowExecutionSettings(ctx); err != nil {
return fmt.Errorf("workflow execution settings: %w", err)
}
if err := b.startRuntimeResourceWatches(ctx, b.runtimeVersions); err != nil {
return fmt.Errorf("runtime resource watches: %w", err)
}
return nil
}
// runtimeResourceKinds are the control-plane kinds this process re-layers
// live: on a new stored version the watch re-runs the layering a reload runs
// (b.runtimeConfig, which applies the execution budgets the settings watch
// published) and republishes through config.Holder, so the change takes
// effect without a restart. Kinds stay out of this list while a
// startup-built component still holds their value -- tool, plugin, container
// and channel settings are frozen in components built at boot and remain
// restart-required -- and a kind joins it only with a consumer that re-reads
// it. Each kind is watched in its own goroutine, exactly the shape the
// workflow-execution-settings watch established.
var runtimeResourceKinds = []string{
controlplane.ProviderSettingsKind,
controlplane.ModelRoleAssignmentsKind,
controlplane.RepositoryPoliciesKind,
controlplane.SchedulingPolicyKind,
}
// startRuntimeResourceWatches keeps a watch per live kind established for the
// life of the process. versions carries the resume point each kind starts
// from, the versions boot's layering recorded. The first stream per kind is
// opened synchronously, so a control plane that cannot be watched at all
// fails the boot that asked for it; after that the watch does not return, it
// reconnects (see keepWatch, whose backoff rules these watches ride).
func (b *boot) startRuntimeResourceWatches(ctx context.Context, versions map[string]int64) error {
for _, kind := range runtimeResourceKinds {
updates, err := b.controlPlane.WatchResource(ctx, kind, versions[kind])
if err != nil {
return fmt.Errorf("watch %s: %w", kind, err)
}
go keepWatch(ctx, b.log, kind, versions[kind], updates,
func(ctx context.Context, afterVersion int64) (<-chan controlplane.AppliedResource, error) {
return b.controlPlane.WatchResource(ctx, kind, afterVersion)
},
waitFor,
func(update controlplane.AppliedResource) int64 { return update.Version },
func(update controlplane.AppliedResource) { b.applyRuntimeResourceUpdate(ctx, kind, update) })
}
return nil
}
// applyRuntimeResourceUpdate re-runs the layering a reload runs and
// republishes the result through config.Holder. The base is the published
// snapshot rather than the boot struct: reloadConfig publishes through the
// same holder and does not update the boot config, so the holder is the one
// view that is current in every path. The layering reads every kind from the
// store over it, so a scheduling-policy document written during this
// process's life and every other kind's last stored value land together, and
// the merge is idempotent (every kind it reads replaces its fields).
//
// The published snapshot is replaced only after the same check the reload
// and the boot layering pass: on a validation failure nothing is published,
// the previous settings keep running (last-known-good), and the failure is
// reported through apply status, which keeps the version still live on the
// record. A refused update therefore reaches the settings page the way a
// refused workflow-execution-settings update does
// (docs/prds/control-plane-apply-status.md).
func (b *boot) applyRuntimeResourceUpdate(ctx context.Context, kind string, update controlplane.AppliedResource) {
if update.Err != nil {
// A stream failure carries no version at all (versions start at 1,
// store.PutResource), and a process must never report an unreachable
// store as its own refusal; the reconnect is keepWatch's business.
if update.Version > 0 {
b.applyStatus.Report(ctx, kind, update.Version, update.Err)
}
b.log.Error("runtime settings watch failed", "kind", kind, "err", update.Err)
return
}
base := b.cfgHolder.Get()
// Boot publishes before the model catalog is merged in, so the snapshot
// this read returns can lack the catalog's model limits and discovered
// providers. The merge is idempotent, and reloadConfig re-applies it
// before layering for the same reason.
applyModelCatalog(&base, b.catalog)
cfg, _, err := b.runtimeConfig(ctx, base)
if err != nil {
// runtimeConfig reported the refusal through apply status for every
// kind it layers, keeping the version each of them last applied.
b.log.Error("runtime settings update rejected; the running settings stay", "kind", kind, "version", update.Version, "err", err)
return
}
b.publishConfig(ctx, cfg)
b.log.Info("runtime settings applied", "kind", kind, "version", update.Version)
switch kind {
case controlplane.ProviderSettingsKind, controlplane.ModelRoleAssignmentsKind:
b.rebuildChatModelRuntime(cfg)
}
}
// rebuildChatModelRuntime re-derives the gateway chat runtime's provider set
// after a live change to provider-settings or model-role-assignments. The
// turn runner reads the runtime through boot.chatLLM, so the swap reaches its
// next turn without rebuilding the runner, and the model manager re-derives
// the references it offers from the new role assignments.
func (b *boot) rebuildChatModelRuntime(cfg config.Config) {
if b.chatModels == nil {
return // this process serves no chat turns
}
b.setLLM(agentexec.NewRuntime(executionProviders(cfg)))
b.chatModels.SetConfigured(cfg.Models)
b.log.Info("chat model runtime rebuilt", "providers", len(cfg.Providers), "models", len(cfg.Models))
}