Files
archie/internal/app/archied/control_plane.go
T

437 lines
20 KiB
Go

package archied
import (
"context"
"encoding/json"
"fmt"
"time"
"github.com/samcharles93/archie-core/internal/app/controlplane"
"github.com/samcharles93/archie-core/internal/app/servicekit"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/domain/workflow"
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
"github.com/samcharles93/archie-core/internal/infrastructure/modelcatalog"
)
// runtimeConfig layers every database-owned setting over a file config and
// validates the result. Boot and the SIGHUP reload both go through it: the
// reload re-resolves the file document alone, so republishing that document
// as it stands reverts each of these layers to its file value until the
// process restarts.
func (b *boot) runtimeConfig(ctx context.Context, base config.Config) (config.Config, map[string]int64, error) {
return servicekit.RuntimeConfig(ctx, b.controlPlane, b.applyStatus, b.secrets, b.log, base, func(cfg *config.Config) {
if settings := b.executionSettings.Load(); settings != nil {
applyExecutionBudgets(cfg, *settings)
}
}, nil)
}
func (b *boot) loadRuntimeConfig(ctx context.Context) error {
cfg, versions, err := b.runtimeConfig(ctx, b.cfg)
if err != nil {
return err
}
b.cfg = cfg
b.cfgHolder.Set(cfg)
// The versions boot layered in are the resume points the runtime-resource
// watches start from: a watch re-established after a version it already
// applied neither replays a document this process handled nor skips one
// it has not.
b.runtimeVersions = versions
return nil
}
// reloadConfig publishes a freshly resolved file document as the running
// config. The database layer is re-applied first, and a control plane that
// cannot answer aborts the reload: the running config already holds those
// values, so publishing the file's over them is the worse outcome.
func (b *boot) reloadConfig(ctx context.Context, doc *configuration.Document) error {
// Boot merges catalog-discovered providers into cfg before the
// Holders are seeded; publishing the raw reloaded document would
// drop them from the running config even though the file is
// unchanged. Re-apply the same merge (idempotent: a catalog that
// failed to load merges to identity).
catalog := b.catalogState()
modelcatalog.Apply(&doc.Config, catalog)
// Bounded: this runs on the signal loop, which handles nothing else
// while it waits, and a State Store that has stopped answering must
// surface as a failed reload rather than a SIGHUP that never returns.
queryCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
cfg, _, err := b.runtimeConfig(queryCtx, doc.Config)
if err != nil {
return fmt.Errorf("runtime settings unavailable: %w", err)
}
old := b.cfgHolder.Get()
restartFields := changedNonReloadableFields(old, cfg)
cfg = preserveForgeIdentity(old, cfg)
doc.Config = cfg
b.currentProvenance.Store(&doc.Provenance)
b.publishConfig(ctx, cfg)
// The container pool reads the published config on every acquire, but the
// dispatcher holds its limit; a reloaded containers.max_concurrency must
// reach it the same way a control-plane update does.
b.resizeTaskDispatcher(cfg.Containers.MaxConcurrency)
if fields := restartFields; len(fields) > 0 {
b.log.Warn("config reloaded; some changes require a restart",
"fields", fields, "paths", doc.Provenance.Paths())
} else {
b.log.Info("config reloaded", "paths", doc.Provenance.Paths())
}
return nil
}
// startWorkflowExecutionSettings loads the stored limits, applies them and
// keeps the stream that delivers later ones established. The first read and
// the first stream are synchronous, so a control plane that cannot be reached
// fails the boot that asked for it; after that the watch does not return, it
// reconnects (see servicekit.KeepWatch).
func (b *boot) startWorkflowExecutionSettings(ctx context.Context) error {
settings, version, err := b.controlPlane.WorkflowExecutionSettings(ctx)
if err != nil {
return err
}
if err := b.applyWorkflowExecutionSettings(ctx, settings, version); err != nil {
return fmt.Errorf("apply workflow execution settings: %w", err)
}
updates, err := b.controlPlane.WatchWorkflowExecutionSettings(ctx, version)
if err != nil {
return err
}
var lastApplyErr, streamErr error
open := func(ctx context.Context, after int64) (<-chan controlplane.AppliedSettings, error) {
updates, err := b.controlPlane.WatchWorkflowExecutionSettings(ctx, after)
if err == nil {
b.applyStatus.Report(ctx, controlplane.WorkflowExecutionSettingsKind, after, lastApplyErr)
}
return updates, err
}
go servicekit.KeepWatch(ctx, b.log, controlplane.WorkflowExecutionSettingsKind, version, updates,
open,
servicekit.WaitFor,
func(update controlplane.AppliedSettings) int64 { return update.Version },
func(update controlplane.AppliedSettings) {
if update.Err != nil {
// A refused document carries its version, while a stream failure
// does not. Keep a refusal visible across a transport outage; when
// there is no outstanding refusal, the end callback records the
// outage until a watch is re-established.
if update.Version > 0 {
lastApplyErr = update.Err
b.applyStatus.Report(ctx, controlplane.WorkflowExecutionSettingsKind, update.Version, update.Err)
} else {
streamErr = update.Err
}
b.log.Error("workflow execution settings watch failed", "err", update.Err)
return
}
lastApplyErr = b.applyWorkflowExecutionSettings(ctx, update.Settings, update.Version)
},
func() {
if lastApplyErr != nil {
return
}
if streamErr == nil {
streamErr = fmt.Errorf("control-plane watch stream ended")
}
b.applyStatus.Report(ctx, controlplane.WorkflowExecutionSettingsKind, 0,
fmt.Errorf("workflow execution settings watch unavailable: %w", streamErr))
streamErr = nil
})
return nil
}
// executionSettingsCandidate is a live workflow-execution-settings update that
// has been built and checked but not yet switched in: the limits themselves,
// and the configuration snapshot every new task would be built from.
type executionSettingsCandidate struct {
settings workflow.ExecutionSettings
cfg config.Config
}
// buildExecutionSettingsCandidate stages a live update without touching
// anything running: it builds the configuration snapshot the new limits would
// be published as and runs this process's own runnability check on it -- the
// same configuration.Validate that runtimeConfig applies before publishing a
// layered config, and that the readiness probe applies to the published one. A
// snapshot this process cannot run is refused before it becomes the live one.
//
// The kind's schema is deliberately not re-checked here. The store validates
// every document before it is written (Definition.Decode) and
// controlplane.Client validates it again as it decodes it, so re-running
// workflow.ExecutionSettings.Validate at this point would be a guard no
// producer can trip.
func (b *boot) buildExecutionSettingsCandidate(settings workflow.ExecutionSettings) (executionSettingsCandidate, error) {
cfg := b.cfgHolder.Get().Clone()
applyExecutionBudgets(&cfg, settings)
if err := configuration.Validate(&cfg); err != nil {
return executionSettingsCandidate{}, fmt.Errorf("workflow execution settings: %w", err)
}
return executionSettingsCandidate{settings: settings, cfg: cfg}, nil
}
// applyWorkflowExecutionSettings builds a candidate and switches it in.
//
// Both halves of the running component move together and only after the check
// passes: on a refusal the previous settings keep running, the configuration
// snapshot new tasks are built from is left as it is, and the failure is
// reported for the version that was refused. The apply-status reporter writes
// that record against the version it last applied, so once this process has
// applied anything the settings page shows a rejected edit rather than a
// version this process never ran.
//
// The settings are recorded as well as applied: they arrive on a watch rather
// than in the file document, so a reload has nowhere else to read them back
// from.
//
// The returned error is propagated by boot, the caller that cannot continue on
// limits it did not apply. Boot reaches this function only with settings the
// control plane already decoded and over a snapshot loadRuntimeConfig already
// validated, so that propagation is the fail-closed direction of the rule
// above rather than a branch the daemon takes today.
func (b *boot) applyWorkflowExecutionSettings(ctx context.Context, settings workflow.ExecutionSettings, version int64) error {
candidate, err := b.buildExecutionSettingsCandidate(settings)
if err != nil {
b.applyStatus.Report(ctx, controlplane.WorkflowExecutionSettingsKind, version, err)
b.log.Error("workflow execution settings rejected; the running settings stay", "version", version, "err", err)
return err
}
b.executionSettings.Store(&candidate.settings)
b.publishConfig(ctx, candidate.cfg)
b.applyStatus.Report(ctx, controlplane.WorkflowExecutionSettingsKind, version, nil)
b.log.Info("workflow execution settings applied", "version", version)
if limit, uptime := settings.MaxTaskRuntime, candidate.cfg.Containers.MaxUptime.Std(); uptime > 0 && (limit <= 0 || limit > uptime) {
b.log.Warn("containers.max_uptime is shorter than the task time limit; the container is killed first", "max_uptime", uptime, "max_task_runtime", limit)
}
return nil
}
func applyExecutionBudgets(cfg *config.Config, settings workflow.ExecutionSettings) {
cfg.Budgets = config.Budgets{
MaxSteps: settings.MaxModelToolSteps, WallClock: config.Duration(settings.MaxRuntime), GateMaxFailures: settings.MaxConsecutiveGateFailures,
TaskWallClock: config.Duration(settings.MaxTaskRuntime),
}
}
// startLiveSettings starts everything a live control-plane update needs
// established before Run continues: the workflow-execution-settings watch,
// whose limits the layering applies, and one watch per runtime-resource kind
// (control_plane.go, runtimeResourceKinds). Both first reads are synchronous,
// so a control plane that cannot be watched at all fails the boot that asked
// for it rather than leaving the process running settings it can no longer
// update.
func (b *boot) startLiveSettings(ctx context.Context) error {
if err := b.startWorkflowExecutionSettings(ctx); err != nil {
return fmt.Errorf("workflow execution settings: %w", err)
}
if err := b.startRuntimeResourceWatches(ctx, b.runtimeVersions, runtimeResourceKinds); err != nil {
return fmt.Errorf("runtime resource watches: %w", err)
}
return nil
}
// runtimeResourceKinds are the control-plane kinds this process re-layers
// live: on a new stored version the watch re-runs the layering a reload runs
// (b.runtimeConfig, which applies the execution budgets the settings watch
// published) and republishes through config.Holder, so the change takes
// effect without a restart. Kinds stay out of this list while a
// startup-built component still holds their value; none does today, so the
// list is every control-plane kind. A kind joins it only with a consumer that
// re-reads it. ToolSettingsKind is on it because applyRuntimeResourceUpdate
// reconciles the MCP provider set and rebuilds the two config-derived tool
// entries (boot.reconcileToolSettings, its consumer); a removed MCP server
// really disconnects, because it is a child process with a Stop. ContainerRuntimePoliciesKind is on it because both consumers
// re-read the published config: the container pool takes its image, pull
// policy, network, max-uptime and concurrency cap per acquire, and the daemon's
// dispatcher is resized after the publish (applyRuntimeResourceUpdate below).
// PluginSettingsKind is on the list because its only consumer is the directory
// reconciliation, which reads the running config each tick; a removed plugin
// or engine still cannot unload, so that kind reports the removal rather than
// requiring a restart. ChannelSettingsKind is on the list for this process's
// own re-layer; the Messaging Service, which owns the channel transports,
// reconciles them separately and restarts only the channel whose settings
// changed, and the Gateway's chat surfaces stay boot-built, so the kind is
// labelled restart-required. AgentProfilesKind is on it because a task's
// profile is resolved per task from the published config (daemon.pinTaskProfile,
// like the container pool's per-acquire read), so a stored change reaches the
// next task without a restart. PluginSettingsKind stays on the list to keep
// its one field, skills_dir, out of the republished config while the running
// directory consumers hold the boot value: the watch refuses such a change
// rather than layering it (refuseSkillsDirChange), so the kind is labelled
// restart-required. Each kind is watched in its own goroutine, exactly the
// shape the workflow-execution-settings watch established. ForgeConnectionsKind
// is on it because the forge client and the worktree token are re-resolved
// from the republished config by reopenForge, which replaces every holder of
// the client (the daemon's root forge and the root task RPC servers), so a
// rotated credential reaches the next operation without a restart.
var runtimeResourceKinds = []string{
controlplane.TaskPolicyKind,
controlplane.ProviderSettingsKind,
controlplane.ModelAliasesKind,
controlplane.RepositoryPoliciesKind,
controlplane.SchedulingPolicyKind,
controlplane.ToolSettingsKind,
controlplane.PluginSettingsKind,
controlplane.ChannelSettingsKind,
controlplane.ContainerRuntimePoliciesKind,
controlplane.CredentialBindingsKind,
controlplane.IdentityGrantsKind,
controlplane.AgentProfileKind,
controlplane.ForgeConnectionsKind,
}
// startRuntimeResourceWatches keeps a watch per live kind established for the
// life of the process. versions carries the resume point each kind starts
// from, the versions boot's layering recorded. The first stream per kind is
// opened synchronously, so a control plane that cannot be watched at all
// fails the boot that asked for it; after that the watch does not return, it
// reconnects (see servicekit.KeepWatch, whose backoff rules these watches ride).
func (b *boot) startRuntimeResourceWatches(ctx context.Context, versions map[string]int64, kinds []string) error {
for _, kind := range kinds {
updates, err := b.controlPlane.WatchResource(ctx, kind, versions[kind])
if err != nil {
return fmt.Errorf("watch %s: %w", kind, err)
}
var lastApplyErr, streamErr error
open := func(ctx context.Context, afterVersion int64) (<-chan controlplane.AppliedResource, error) {
next, err := b.controlPlane.WatchResource(ctx, kind, afterVersion)
if err == nil {
// A live watch is re-established: the outage clears, and an
// outstanding refusal stays the record's error.
b.applyStatus.Report(ctx, kind, afterVersion, lastApplyErr)
}
return next, err
}
go servicekit.KeepWatch(ctx, b.log, kind, versions[kind], updates,
open,
servicekit.WaitFor,
func(update controlplane.AppliedResource) int64 { return update.Version },
func(update controlplane.AppliedResource) {
if update.Err != nil {
// A stream failure carries no version at all, and a process
// must never report an unreachable store as its own
// refusal: the outage is what the end callback records.
if update.Version > 0 {
lastApplyErr = update.Err
b.applyStatus.Report(ctx, kind, update.Version, update.Err)
} else {
streamErr = update.Err
}
b.log.Error("runtime settings watch failed", "kind", kind, "err", update.Err)
return
}
lastApplyErr = b.applyRuntimeResourceUpdate(ctx, kind, update)
},
func() {
// A refusal already on the record outranks the outage; it is the
// state the running settings are actually in. Otherwise the gap is
// reported so the record stops reading the last-applied version as
// current while no live updates can arrive.
if lastApplyErr != nil {
return
}
if streamErr == nil {
streamErr = fmt.Errorf("control-plane watch stream ended")
}
b.applyStatus.Report(ctx, kind, 0,
fmt.Errorf("runtime settings watch unavailable: %w", streamErr))
streamErr = nil
})
}
return nil
}
// applyRuntimeResourceUpdate re-runs the layering a reload runs and
// republishes the result through config.Holder. The base is the published
// snapshot rather than the boot struct: reloadConfig publishes through the
// same holder and does not update the boot config, so the holder is the one
// view that is current in every path. The layering reads every kind from the
// store over it, so a scheduling-policy document written during this
// process's life and every other kind's last stored value land together, and
// the merge is idempotent (every kind it reads replaces its fields).
//
// The published snapshot is replaced only after the same check the reload
// and the boot layering pass: on a validation failure nothing is published,
// the previous settings keep running (last-known-good), and the failure is
// reported through apply status, which keeps the version still live on the
// record. A refused update therefore reaches the settings page the way a
// refused workflow-execution-settings update does.
//
// The returned error is the refusal or rejection, for the watch to keep as
// lastApplyErr: it is what keeps the outage callback from overwriting a
// refusal the record already shows with a transport gap.
func (b *boot) applyRuntimeResourceUpdate(ctx context.Context, kind string, update controlplane.AppliedResource) error {
if kind == controlplane.PluginSettingsKind {
if err := b.refuseSkillsDirChange(ctx); err != nil {
b.applyStatus.Report(ctx, kind, update.Version, err)
b.log.Error("plugin settings refused; the running settings stay", "version", update.Version, "err", err)
return err
}
}
base := b.cfgHolder.Get()
// Boot publishes before the model catalog is merged in, so the snapshot
// this read returns can lack the catalog's model limits and discovered
// providers. The merge is idempotent, and reloadConfig re-applies it
// before layering for the same reason.
catalog := b.catalogState()
modelcatalog.Apply(&base, catalog)
cfg, _, err := b.runtimeConfig(ctx, base)
if err != nil {
// runtimeConfig reported the refusal through apply status for every
// kind it layers, keeping the version each of them last applied.
b.log.Error("runtime settings update rejected; the running settings stay", "kind", kind, "version", update.Version, "err", err)
return err
}
b.publishConfig(ctx, cfg)
b.log.Info("runtime settings applied", "kind", kind, "version", update.Version)
if kind == controlplane.ContainerRuntimePoliciesKind {
// The pool reads the republished config itself; the dispatcher holds
// its limit, so a raise must be woken to admit queued work now and a
// lower must be honoured as running tasks finish.
b.resizeTaskDispatcher(cfg.Containers.MaxConcurrency)
}
if kind == controlplane.ForgeConnectionsKind {
// The forge client is built from the credential rather than taken from
// the published config, so the change that published it must reach the
// client too: a rotated token otherwise keeps authenticating with the
// boot value until the process restarts.
b.reopenForge(ctx)
}
return nil
}
// resizeTaskDispatcher applies a new containers.max_concurrency to the
// daemon's running dispatcher. The pool needs no poke because it reads the
// published config on every acquire. b.d is nil in the boot window in which
// the watches are already live but the daemon is not built yet; there the
// dispatcher is created later from the config the next drain reads.
func (b *boot) resizeTaskDispatcher(maxConcurrency int) {
if b.d != nil {
b.d.ResizeTaskDispatcher(maxConcurrency)
}
}
// refuseSkillsDirChange refuses a stored plugin-settings document whose
// skills_dir differs from the value in force. skills_dir is the one directory
// with no live consumer, so a change to it stays a restart-scoped edit rather
// than a value the daemon reports as applied.
func (b *boot) refuseSkillsDirChange(ctx context.Context) error {
var stored struct {
SkillsDir string `json:"skills_dir"`
}
_, found, err := b.controlPlane.Query(ctx, controlplane.PluginSettingsKind, func(value []byte) error {
return json.Unmarshal(value, &stored)
})
if err != nil {
return fmt.Errorf("read plugin settings: %w", err)
}
if !found {
return nil
}
if running := b.cfgHolder.Get().SkillsDir; stored.SkillsDir != running {
return fmt.Errorf("skills_dir applies on restart: stored %q, running %q", stored.SkillsDir, running)
}
return nil
}