227 lines
10 KiB
Go
227 lines
10 KiB
Go
package archied
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/samcharles93/archie-core/internal/app/controlplane"
|
|
"github.com/samcharles93/archie-core/internal/config"
|
|
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
|
|
)
|
|
|
|
// runtimeConfig layers every database-owned setting over a file config and
|
|
// validates the result. Boot and the SIGHUP reload both go through it: the
|
|
// reload re-resolves the file document alone, so republishing that document
|
|
// as it stands reverts each of these layers to its file value until the
|
|
// process restarts (archie-core-ju85).
|
|
func (b *boot) runtimeConfig(ctx context.Context, base config.Config) (config.Config, error) {
|
|
cfg, versions, err := b.controlPlane.RuntimeConfig(ctx, base)
|
|
if err != nil {
|
|
return config.Config{}, err
|
|
}
|
|
if settings := b.executionSettings.Load(); settings != nil {
|
|
applyExecutionBudgets(&cfg, *settings)
|
|
}
|
|
if err := configuration.Validate(&cfg); err != nil {
|
|
wrapped := fmt.Errorf("validate database settings: %w", err)
|
|
b.reportApplied(ctx, versions, wrapped)
|
|
return config.Config{}, wrapped
|
|
}
|
|
// The State Store persists source references, while openStores resolved
|
|
// only the file snapshot. Layering provider-settings restores the source
|
|
// references, so resolve the effective map before any runtime consumes it.
|
|
if err := resolveProviderSecrets(&cfg, b.secrets, b.log); err != nil {
|
|
b.reportApplied(ctx, versions, err)
|
|
return config.Config{}, err
|
|
}
|
|
b.reportApplied(ctx, versions, nil)
|
|
return cfg, nil
|
|
}
|
|
|
|
// reportApplied publishes the version of each kind this process just layered
|
|
// in. On a validation failure every kind is reported with the error, because
|
|
// the layering is all-or-nothing: none of the versions read took effect.
|
|
func (b *boot) reportApplied(ctx context.Context, versions map[string]int64, applyErr error) {
|
|
for kind, version := range versions {
|
|
b.applyStatus.Report(ctx, kind, version, applyErr)
|
|
}
|
|
}
|
|
|
|
func (b *boot) loadRuntimeConfig(ctx context.Context) error {
|
|
cfg, err := b.runtimeConfig(ctx, b.cfg)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
b.cfg = cfg
|
|
b.cfgHolder.Set(cfg)
|
|
return nil
|
|
}
|
|
|
|
// reloadConfig publishes a freshly resolved file document as the running
|
|
// config. The database layer is re-applied first, and a control plane that
|
|
// cannot answer aborts the reload: the running config already holds those
|
|
// values, so publishing the file's over them is the worse outcome.
|
|
func (b *boot) reloadConfig(ctx context.Context, doc *configuration.Document) error {
|
|
// Boot merges catalog-discovered providers into cfg before the
|
|
// Holders are seeded; publishing the raw reloaded document would
|
|
// drop them from the running config even though the file is
|
|
// unchanged. Re-apply the same merge (idempotent: a catalog that
|
|
// failed to load merges to identity).
|
|
applyModelCatalog(&doc.Config, b.catalog)
|
|
// Bounded: this runs on the signal loop, which handles nothing else
|
|
// while it waits, and a State Store that has stopped answering must
|
|
// surface as a failed reload rather than a SIGHUP that never returns.
|
|
queryCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
|
defer cancel()
|
|
cfg, err := b.runtimeConfig(queryCtx, doc.Config)
|
|
if err != nil {
|
|
return fmt.Errorf("runtime settings unavailable: %w", err)
|
|
}
|
|
doc.Config = cfg
|
|
old := b.cfgHolder.Get()
|
|
b.currentProvenance.Store(&doc.Provenance)
|
|
b.publishConfig(ctx, cfg)
|
|
if fields := changedNonReloadableFields(old, cfg); len(fields) > 0 {
|
|
b.log.Warn("config reloaded; some changes require a restart",
|
|
"fields", fields, "paths", doc.Provenance.Paths())
|
|
} else {
|
|
b.log.Info("config reloaded", "paths", doc.Provenance.Paths())
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// startWorkflowExecutionSettings loads the stored limits, applies them and
|
|
// keeps the stream that delivers later ones established. The first read and
|
|
// the first stream are synchronous, so a control plane that cannot be reached
|
|
// fails the boot that asked for it; after that the watch does not return, it
|
|
// reconnects (see keepWatch).
|
|
func (b *boot) startWorkflowExecutionSettings(ctx context.Context) error {
|
|
settings, version, err := b.controlPlane.WorkflowExecutionSettings(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := b.applyWorkflowExecutionSettings(ctx, settings, version); err != nil {
|
|
return fmt.Errorf("apply workflow execution settings: %w", err)
|
|
}
|
|
updates, err := b.controlPlane.WatchWorkflowExecutionSettings(ctx, version)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
go keepWatch(ctx, b.log, controlplane.WorkflowExecutionSettingsKind, version, updates,
|
|
b.controlPlane.WatchWorkflowExecutionSettings,
|
|
waitFor,
|
|
func(update controlplane.AppliedSettings) int64 { return update.Version },
|
|
func(update controlplane.AppliedSettings) {
|
|
if update.Err != nil {
|
|
// A refused update arrives here, not through
|
|
// applyWorkflowExecutionSettings: controlplane.Client decodes every
|
|
// document it is handed, so a stored value this process cannot run
|
|
// never reaches the apply call. Report it against the version it
|
|
// came from -- the reporter keeps the version it last applied --
|
|
// so the settings page shows the refusal instead of a version this
|
|
// process never ran (docs/prds/control-plane-apply-status.md).
|
|
//
|
|
// A stream failure arrives the same way and must not be reported:
|
|
// it carries no version at all, because a Recv error is the store
|
|
// not answering rather than a document, and versions start at 1
|
|
// (store.PutResource). The rule is
|
|
// docs/prds/control-plane-apply-status.md, "What can be reported,
|
|
// and what cannot": a process reaches its apply point only after
|
|
// the State Store has answered it, and an unreachable store is
|
|
// never reported as a failure by the process it affected. The
|
|
// record would not stick of itself -- Report overwrites it on the
|
|
// next applied version -- but a store that never changes again
|
|
// leaves the one stamp standing, and the process must not write it
|
|
// at all. Reporting it would also overwrite the text of a standing
|
|
// refusal, which is the one message that says what to fix. The
|
|
// reconnect is the loop's business, not the record's.
|
|
if update.Version > 0 {
|
|
b.applyStatus.Report(ctx, controlplane.WorkflowExecutionSettingsKind, update.Version, update.Err)
|
|
}
|
|
b.log.Error("workflow execution settings watch failed", "err", update.Err)
|
|
return
|
|
}
|
|
_ = b.applyWorkflowExecutionSettings(ctx, update.Settings, update.Version)
|
|
})
|
|
return nil
|
|
}
|
|
|
|
// executionSettingsCandidate is a live workflow-execution-settings update that
|
|
// has been built and checked but not yet switched in: the limits themselves,
|
|
// and the configuration snapshot every new task would be built from.
|
|
type executionSettingsCandidate struct {
|
|
settings workflow.ExecutionSettings
|
|
cfg config.Config
|
|
}
|
|
|
|
// buildExecutionSettingsCandidate stages a live update without touching
|
|
// anything running: it builds the configuration snapshot the new limits would
|
|
// be published as and runs this process's own runnability check on it -- the
|
|
// same configuration.Validate that runtimeConfig applies before publishing a
|
|
// layered config, and that the readiness probe applies to the published one. A
|
|
// snapshot this process cannot run is refused before it becomes the live one
|
|
// (docs/prds/runtime-control-plane.md, "API": Archie starts and checks the new
|
|
// one before switching).
|
|
//
|
|
// The kind's schema is deliberately not re-checked here. The store validates
|
|
// every document before it is written (Definition.Decode) and
|
|
// controlplane.Client validates it again as it decodes it, so re-running
|
|
// workflow.ExecutionSettings.Validate at this point would be a guard no
|
|
// producer can trip.
|
|
func (b *boot) buildExecutionSettingsCandidate(settings workflow.ExecutionSettings) (executionSettingsCandidate, error) {
|
|
cfg := b.cfgHolder.Get().Clone()
|
|
applyExecutionBudgets(&cfg, settings)
|
|
if err := configuration.Validate(&cfg); err != nil {
|
|
return executionSettingsCandidate{}, fmt.Errorf("workflow execution settings: %w", err)
|
|
}
|
|
return executionSettingsCandidate{settings: settings, cfg: cfg}, nil
|
|
}
|
|
|
|
// applyWorkflowExecutionSettings builds a candidate and switches it in.
|
|
//
|
|
// Both halves of the running component move together and only after the check
|
|
// passes: on a refusal the previous settings keep running, the configuration
|
|
// snapshot new tasks are built from is left as it is, and the failure is
|
|
// reported for the version that was refused. The apply-status reporter writes
|
|
// that record against the version it last applied, so once this process has
|
|
// applied anything the settings page shows a rejected edit rather than a
|
|
// version this process never ran.
|
|
//
|
|
// The settings are recorded as well as applied: they arrive on a watch rather
|
|
// than in the file document, so a reload has nowhere else to read them back
|
|
// from.
|
|
//
|
|
// The returned error is propagated by boot, the caller that cannot continue on
|
|
// limits it did not apply. Boot reaches this function only with settings the
|
|
// control plane already decoded and over a snapshot loadRuntimeConfig already
|
|
// validated, so that propagation is the fail-closed direction of the rule
|
|
// above rather than a branch the daemon takes today.
|
|
func (b *boot) applyWorkflowExecutionSettings(ctx context.Context, settings workflow.ExecutionSettings, version int64) error {
|
|
candidate, err := b.buildExecutionSettingsCandidate(settings)
|
|
if err != nil {
|
|
b.applyStatus.Report(ctx, controlplane.WorkflowExecutionSettingsKind, version, err)
|
|
b.log.Error("workflow execution settings rejected; the running settings stay", "version", version, "err", err)
|
|
return err
|
|
}
|
|
b.executionSettings.Store(&candidate.settings)
|
|
b.cfgHolder.Set(candidate.cfg)
|
|
b.applyStatus.Report(ctx, controlplane.WorkflowExecutionSettingsKind, version, nil)
|
|
b.log.Info("workflow execution settings applied", "version", version)
|
|
if limit, uptime := settings.MaxTaskRuntime, candidate.cfg.Containers.MaxUptime.Std(); uptime > 0 && (limit <= 0 || limit > uptime) {
|
|
b.log.Warn("containers.max_uptime is shorter than the task time limit; the container is killed first", "max_uptime", uptime, "max_task_runtime", limit)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func applyExecutionBudgets(cfg *config.Config, settings workflow.ExecutionSettings) {
|
|
cfg.Budgets = config.Budgets{
|
|
MaxSteps: settings.MaxModelToolSteps, WallClock: config.Duration(settings.MaxRuntime), GateMaxFailures: settings.MaxConsecutiveGateFailures,
|
|
TaskWallClock: config.Duration(settings.MaxTaskRuntime),
|
|
}
|
|
for i := range cfg.Identities {
|
|
cfg.Identities[i].Budgets = cfg.Budgets
|
|
}
|
|
}
|