Files
archie/internal/app/archied/control_plane.go
T

227 lines
10 KiB
Go

package archied
import (
"context"
"fmt"
"time"
"github.com/samcharles93/archie-core/internal/app/controlplane"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/domain/workflow"
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
)
// runtimeConfig layers every database-owned setting over a file config and
// validates the result. Boot and the SIGHUP reload both go through it: the
// reload re-resolves the file document alone, so republishing that document
// as it stands reverts each of these layers to its file value until the
// process restarts (archie-core-ju85).
func (b *boot) runtimeConfig(ctx context.Context, base config.Config) (config.Config, error) {
cfg, versions, err := b.controlPlane.RuntimeConfig(ctx, base)
if err != nil {
return config.Config{}, err
}
if settings := b.executionSettings.Load(); settings != nil {
applyExecutionBudgets(&cfg, *settings)
}
if err := configuration.Validate(&cfg); err != nil {
wrapped := fmt.Errorf("validate database settings: %w", err)
b.reportApplied(ctx, versions, wrapped)
return config.Config{}, wrapped
}
// The State Store persists source references, while openStores resolved
// only the file snapshot. Layering provider-settings restores the source
// references, so resolve the effective map before any runtime consumes it.
if err := resolveProviderSecrets(&cfg, b.secrets, b.log); err != nil {
b.reportApplied(ctx, versions, err)
return config.Config{}, err
}
b.reportApplied(ctx, versions, nil)
return cfg, nil
}
// reportApplied publishes the version of each kind this process just layered
// in. On a validation failure every kind is reported with the error, because
// the layering is all-or-nothing: none of the versions read took effect.
func (b *boot) reportApplied(ctx context.Context, versions map[string]int64, applyErr error) {
for kind, version := range versions {
b.applyStatus.Report(ctx, kind, version, applyErr)
}
}
func (b *boot) loadRuntimeConfig(ctx context.Context) error {
cfg, err := b.runtimeConfig(ctx, b.cfg)
if err != nil {
return err
}
b.cfg = cfg
b.cfgHolder.Set(cfg)
return nil
}
// reloadConfig publishes a freshly resolved file document as the running
// config. The database layer is re-applied first, and a control plane that
// cannot answer aborts the reload: the running config already holds those
// values, so publishing the file's over them is the worse outcome.
func (b *boot) reloadConfig(ctx context.Context, doc *configuration.Document) error {
// Boot merges catalog-discovered providers into cfg before the
// Holders are seeded; publishing the raw reloaded document would
// drop them from the running config even though the file is
// unchanged. Re-apply the same merge (idempotent: a catalog that
// failed to load merges to identity).
applyModelCatalog(&doc.Config, b.catalog)
// Bounded: this runs on the signal loop, which handles nothing else
// while it waits, and a State Store that has stopped answering must
// surface as a failed reload rather than a SIGHUP that never returns.
queryCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
cfg, err := b.runtimeConfig(queryCtx, doc.Config)
if err != nil {
return fmt.Errorf("runtime settings unavailable: %w", err)
}
doc.Config = cfg
old := b.cfgHolder.Get()
b.currentProvenance.Store(&doc.Provenance)
b.publishConfig(ctx, cfg)
if fields := changedNonReloadableFields(old, cfg); len(fields) > 0 {
b.log.Warn("config reloaded; some changes require a restart",
"fields", fields, "paths", doc.Provenance.Paths())
} else {
b.log.Info("config reloaded", "paths", doc.Provenance.Paths())
}
return nil
}
// startWorkflowExecutionSettings loads the stored limits, applies them and
// keeps the stream that delivers later ones established. The first read and
// the first stream are synchronous, so a control plane that cannot be reached
// fails the boot that asked for it; after that the watch does not return, it
// reconnects (see keepWatch).
func (b *boot) startWorkflowExecutionSettings(ctx context.Context) error {
settings, version, err := b.controlPlane.WorkflowExecutionSettings(ctx)
if err != nil {
return err
}
if err := b.applyWorkflowExecutionSettings(ctx, settings, version); err != nil {
return fmt.Errorf("apply workflow execution settings: %w", err)
}
updates, err := b.controlPlane.WatchWorkflowExecutionSettings(ctx, version)
if err != nil {
return err
}
go keepWatch(ctx, b.log, controlplane.WorkflowExecutionSettingsKind, version, updates,
b.controlPlane.WatchWorkflowExecutionSettings,
waitFor,
func(update controlplane.AppliedSettings) int64 { return update.Version },
func(update controlplane.AppliedSettings) {
if update.Err != nil {
// A refused update arrives here, not through
// applyWorkflowExecutionSettings: controlplane.Client decodes every
// document it is handed, so a stored value this process cannot run
// never reaches the apply call. Report it against the version it
// came from -- the reporter keeps the version it last applied --
// so the settings page shows the refusal instead of a version this
// process never ran (docs/prds/control-plane-apply-status.md).
//
// A stream failure arrives the same way and must not be reported:
// it carries no version at all, because a Recv error is the store
// not answering rather than a document, and versions start at 1
// (store.PutResource). The rule is
// docs/prds/control-plane-apply-status.md, "What can be reported,
// and what cannot": a process reaches its apply point only after
// the State Store has answered it, and an unreachable store is
// never reported as a failure by the process it affected. The
// record would not stick of itself -- Report overwrites it on the
// next applied version -- but a store that never changes again
// leaves the one stamp standing, and the process must not write it
// at all. Reporting it would also overwrite the text of a standing
// refusal, which is the one message that says what to fix. The
// reconnect is the loop's business, not the record's.
if update.Version > 0 {
b.applyStatus.Report(ctx, controlplane.WorkflowExecutionSettingsKind, update.Version, update.Err)
}
b.log.Error("workflow execution settings watch failed", "err", update.Err)
return
}
_ = b.applyWorkflowExecutionSettings(ctx, update.Settings, update.Version)
})
return nil
}
// executionSettingsCandidate is a live workflow-execution-settings update that
// has been built and checked but not yet switched in: the limits themselves,
// and the configuration snapshot every new task would be built from.
type executionSettingsCandidate struct {
settings workflow.ExecutionSettings
cfg config.Config
}
// buildExecutionSettingsCandidate stages a live update without touching
// anything running: it builds the configuration snapshot the new limits would
// be published as and runs this process's own runnability check on it -- the
// same configuration.Validate that runtimeConfig applies before publishing a
// layered config, and that the readiness probe applies to the published one. A
// snapshot this process cannot run is refused before it becomes the live one
// (docs/prds/runtime-control-plane.md, "API": Archie starts and checks the new
// one before switching).
//
// The kind's schema is deliberately not re-checked here. The store validates
// every document before it is written (Definition.Decode) and
// controlplane.Client validates it again as it decodes it, so re-running
// workflow.ExecutionSettings.Validate at this point would be a guard no
// producer can trip.
func (b *boot) buildExecutionSettingsCandidate(settings workflow.ExecutionSettings) (executionSettingsCandidate, error) {
cfg := b.cfgHolder.Get().Clone()
applyExecutionBudgets(&cfg, settings)
if err := configuration.Validate(&cfg); err != nil {
return executionSettingsCandidate{}, fmt.Errorf("workflow execution settings: %w", err)
}
return executionSettingsCandidate{settings: settings, cfg: cfg}, nil
}
// applyWorkflowExecutionSettings builds a candidate and switches it in.
//
// Both halves of the running component move together and only after the check
// passes: on a refusal the previous settings keep running, the configuration
// snapshot new tasks are built from is left as it is, and the failure is
// reported for the version that was refused. The apply-status reporter writes
// that record against the version it last applied, so once this process has
// applied anything the settings page shows a rejected edit rather than a
// version this process never ran.
//
// The settings are recorded as well as applied: they arrive on a watch rather
// than in the file document, so a reload has nowhere else to read them back
// from.
//
// The returned error is propagated by boot, the caller that cannot continue on
// limits it did not apply. Boot reaches this function only with settings the
// control plane already decoded and over a snapshot loadRuntimeConfig already
// validated, so that propagation is the fail-closed direction of the rule
// above rather than a branch the daemon takes today.
func (b *boot) applyWorkflowExecutionSettings(ctx context.Context, settings workflow.ExecutionSettings, version int64) error {
candidate, err := b.buildExecutionSettingsCandidate(settings)
if err != nil {
b.applyStatus.Report(ctx, controlplane.WorkflowExecutionSettingsKind, version, err)
b.log.Error("workflow execution settings rejected; the running settings stay", "version", version, "err", err)
return err
}
b.executionSettings.Store(&candidate.settings)
b.cfgHolder.Set(candidate.cfg)
b.applyStatus.Report(ctx, controlplane.WorkflowExecutionSettingsKind, version, nil)
b.log.Info("workflow execution settings applied", "version", version)
if limit, uptime := settings.MaxTaskRuntime, candidate.cfg.Containers.MaxUptime.Std(); uptime > 0 && (limit <= 0 || limit > uptime) {
b.log.Warn("containers.max_uptime is shorter than the task time limit; the container is killed first", "max_uptime", uptime, "max_task_runtime", limit)
}
return nil
}
func applyExecutionBudgets(cfg *config.Config, settings workflow.ExecutionSettings) {
cfg.Budgets = config.Budgets{
MaxSteps: settings.MaxModelToolSteps, WallClock: config.Duration(settings.MaxRuntime), GateMaxFailures: settings.MaxConsecutiveGateFailures,
TaskWallClock: config.Duration(settings.MaxTaskRuntime),
}
for i := range cfg.Identities {
cfg.Identities[i].Budgets = cfg.Budgets
}
}