From 04fa1b74fb717a4e80ee0d6082b340bfdcf876b3 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Fri, 2 Oct 2026 11:28:34 +1000 Subject: [PATCH] feat(container): apply container-runtime-policies live The container runtime policies kind was restart-required because the pool captured image, pull policy, network, max uptime and max concurrency at construction and the dispatcher fixed its slot count at the start of a drain pass. Both consumers now re-read the published config: - the pool's settings come from a live source read per acquire, with the boot-resolved registry credential and Docker construction values preserved; a live default-image change is pulled rather than assumed present; - the daemon owns one long-lived dispatcher that accepts a new limit: running tasks keep their slots, a raise admits queued work immediately, and a lower limit is applied as tasks finish; - the control-plane descriptor is live, the kind joins runtimeResourceKinds, and the apply/reload paths resize the dispatcher after publishing. The ContainerConfig fields whose consumers now re-read are added to the reload allowlist; registry_auth stays restart-required. Regenerate the docs artifacts. contracts.json also carries the voice media-type description a prior commit left stale. --- .../skills/archie-config-and-flags/SKILL.md | 16 +- docs/architecture/configuration.md | 33 ++-- docs/data/generated/contracts.json | 2 +- docs/data/generated/dev-docs.json | 2 +- internal/app/archied/bootstrap.go | 2 +- internal/app/archied/bootstrap_nats_test.go | 2 +- internal/app/archied/control_plane.go | 29 +++- .../archied/control_plane_resources_test.go | 31 ++++ internal/app/archied/main.go | 25 ++- internal/app/archied/reload.go | 20 ++- internal/app/archied/reload_test.go | 21 ++- .../archied/state_store_regression_test.go | 2 +- .../app/controlplane/controlplane_test.go | 4 +- .../app/controlplane/operational_settings.go | 8 +- .../app/controlplane/runtime_config_test.go | 4 +- internal/container/network_test.go | 4 +- internal/container/pool.go | 127 ++++++++++---- internal/container/pool_test.go | 156 ++++++++++++++++++ internal/daemon/container_env_test.go | 2 +- internal/daemon/daemon.go | 81 +++++++-- internal/daemon/task_cancellation_test.go | 2 +- internal/daemon/task_dispatcher_live_test.go | 103 ++++++++++++ internal/daemon/terminal_cleanup_test.go | 2 +- 23 files changed, 573 insertions(+), 105 deletions(-) create mode 100644 internal/daemon/task_dispatcher_live_test.go diff --git a/.agents/skills/archie-config-and-flags/SKILL.md b/.agents/skills/archie-config-and-flags/SKILL.md index a9ae97488..87eca6959 100644 --- a/.agents/skills/archie-config-and-flags/SKILL.md +++ b/.agents/skills/archie-config-and-flags/SKILL.md @@ -170,14 +170,14 @@ looking at before reporting one: | Key | Effective behaviour | | ----------------- | ------------------------------------------------------------------------------------------------------------------ | -| `image` | Required only when enabled. | -| `max_concurrency` | Zero means no limit; used by general task dispatcher and container pool. | -| `max_uptime` | Default `60m` only when enabled; caps container lifetime. | -| `volume_ttl` | Default `72h` only when enabled; negative rejected. | -| `pull_policy` | Default `missing` when enabled. Pool code recognises `missing` and `always`; other strings silently skip pre-pull. | -| `registry_auth` | `SecretRef` (file-owned, `json:"-"`). Resolved at boot by `archied`'s `resolveRegistryAuth` and passed to `container.Config.RegistryAuth`; the pool base64url-encodes it into the `X-Registry-Auth` header (`internal/container/pool.go`'s `registryAuthHeader`). Zero ref = anonymous pull. Unresolvable/empty credential or an unencodable one warns and pulls anonymously rather than failing the pool. A half-named ref is refused by `configuration.validateContainers`. Preserved across the `container-runtime-policies` layering (the document cannot carry a secret ref), unlike `image`/`pull_policy`. | -| `network` | Empty uses best-effort self-network detection. Repository overlay pins `archie-core_default`. | -| `profiles` | Named agent execution environments (image or Kit); resolved per task dispatch (`ContainerConfig.Profile`). Own control-plane resource `agent-profiles` (`json:"-"` on the field so `container-runtime-policies` never carries it), `ApplyMode: live` -- a profile change applies without a restart, unlike the rest of `[containers]`. | +| `image` | Required at boot. Read by the container pool on every acquire (`containerLiveSettings` -> `Pool.current`, archie-core-zfb0.2), so a stored change applies without a restart. | +| `max_concurrency` | Zero means no limit. Read by the pool per acquire and applied to the daemon's running task dispatcher on publish (`ResizeTaskDispatcher`). | +| `max_uptime` | Default `60m` at boot; caps container lifetime. Read by the pool per acquire, so a stored change reaps the next container under the new cap. | +| `volume_ttl` | Default `72h` only when enabled; negative rejected. Re-read per cleanup cycle. | +| `pull_policy` | Default `missing` at boot. Pool code recognises `missing` and `always`; other strings silently skip pre-pull. Read by the pool per acquire. | +| `registry_auth` | `SecretRef` (file-owned, `json:"-"`). Resolved at boot by `archied`'s `resolveRegistryAuth` and passed to `container.Config.RegistryAuth`; the pool base64url-encodes it into the `X-Registry-Auth` header (`internal/container/pool.go`'s `registryAuthHeader`). Zero ref = anonymous pull. Unresolvable/empty credential or an unencodable one warns and pulls anonymously rather than failing the pool. A half-named ref is refused by `configuration.validateContainers`. `container-runtime-policies` cannot carry a secret ref, so this stays boot-resolved and requires-restart, unlike `image`/`pull_policy`. | +| `network` | Empty uses best-effort self-network detection at boot; the repository overlay pins `archie-core_default`. Read by the pool per acquire, falling back to the boot-resolved network when the live value is empty. | +| `profiles` | Named agent execution environments (image or Kit); resolved per task dispatch (`ContainerConfig.Profile`). Own control-plane resource `agent-profiles` (`json:"-"` on the field so `container-runtime-policies` never carries it), `ApplyMode: live`. | | `credentials` | Maps a Kit `credential@1` service to a secret (`config.CredentialBinding`). Own control-plane resource `credential-bindings`, `ApplyMode: live`, `json:"-"` on the field. `kitrun.Launcher` reads it fresh from the live `config.Holder` on every `Launch`, never a value captured at boot -- see `internal/infrastructure/kitrun/launcher.go`'s `resolveCredentials`. A binding alone grants nothing; see `org`/`granted_credentials` below. | | `org` / `granted_credentials` (root and per-identity) | An identity's org and the `credentials` service names it may resolve. `daemon.runKitTask` reads both fresh per dispatch (`d.configFor(task)`) and passes them to `kitrun.Request`; `config.ContainerConfig.BoundCredentials` computes the declared (Kit) ∩ granted (identity) ∩ org-matched intersection -- neither declared nor granted widens the other. | diff --git a/docs/architecture/configuration.md b/docs/architecture/configuration.md index ff93c199a..2bf507c31 100644 --- a/docs/architecture/configuration.md +++ b/docs/architecture/configuration.md @@ -335,21 +335,28 @@ Two properties matter to anyone changing this area: query, so boot records the last published settings and the reload re-applies them from there. - **Live kinds arrive on a watch.** Provider settings, model role assignments, - repository policies, scheduling policy, plugin settings, review settings and - channel settings are `ApplyMode: live`: the daemon keeps a watch per kind and, - on a new version, re-runs `boot.runtimeConfig` over the published snapshot and + repository policies, scheduling policy, plugin settings, review settings, + channel settings and container runtime policies are `ApplyMode: live`: the + daemon keeps a watch per kind (`boot.startRuntimeResourceWatches`, + `runtimeResourceKinds`) and, on a new version, re-runs `boot.runtimeConfig` + over the published snapshot and republishes through `config.Holder`. A candidate the layered document cannot run is refused before it is published — last-known-good keeps running and - the refusal is reported through apply status. A kind whose consumer lives in - another process reconciles there too: `archie-messaging` re-reads the stored - channel settings on the apply-status restamp interval and restarts only the - channel whose transport settings changed, while the listen addresses it has - bound and the set of channels it composed stay process bindings that take - effect on restart. A kind joins the live list only with a consumer that - re-reads it; one whose startup-built consumer never does stays - restart-required, and the model/provider half of the live set also rebuilds - the gateway chat runtime's provider set and the chat model list the chat - surfaces offer. + the refusal is reported through apply status. Container runtime policies + join the live set because their consumers re-read the published config: the + container pool reads image, pull policy, network, max uptime and concurrency + cap on every acquire and the daemon resizes its running task dispatcher from + the same publish; the fields the document cannot carry, `registry_auth` + above all, stay boot-resolved and remain requires-restart. A kind whose + consumer lives in another process reconciles there too: `archie-messaging` + re-reads the stored channel settings on the apply-status restamp interval + and restarts only the channel whose transport settings changed, while the + listen addresses it has bound and the set of channels it composed stay + process bindings that take effect on restart. A kind joins the live list + only with a consumer that re-reads it; one whose startup-built consumer + never does stays restart-required, and the model/provider half of the live + set also rebuilds the gateway chat runtime's provider set and the chat model + list the chat surfaces offer. ## Completion criteria diff --git a/docs/data/generated/contracts.json b/docs/data/generated/contracts.json index e46e00278..24753a052 100644 --- a/docs/data/generated/contracts.json +++ b/docs/data/generated/contracts.json @@ -350,7 +350,7 @@ "type": "object" }, "type": { - "description": "Type is \"image\", \"video\", \"audio\", or \"document\".", + "description": "Type is \"image\", \"video\", \"audio\", \"voice\", or \"document\".", "type": "string" }, "url": { diff --git a/docs/data/generated/dev-docs.json b/docs/data/generated/dev-docs.json index 46b5d2594..5aae1a858 100644 --- a/docs/data/generated/dev-docs.json +++ b/docs/data/generated/dev-docs.json @@ -28,7 +28,7 @@ "url": "/dev/architecture/configuration/", "title": "Configuration", "section": "Architecture", - "body": "# Configuration\n\n**Status:** Approved; existing-type assignment is ongoing \n**Date:** 2026-07-28 \n**Tracking issue:** [#73](https://github.com/samcharles93/archie-core/issues/73)\n\n## Ownership\n\nConfiguration has two separate responsibilities:\n\n1. `internal/infrastructure/configuration` loads files, environment variables,\n secret references, overlays, and compatibility representations.\n2. Each domain and plugin owns the typed runtime settings required for its\n behaviour.\n\n`internal/app` translates external input into those settings, validates\ncross-capability combinations, and supplies each component only its own\nsettings.\n\nThe following are prohibited:\n\n- passing complete application configuration into a domain;\n- making a domain depend on TOML, YAML, environment-variable names, secret\n references, or a configuration library;\n- defining a central runtime configuration type imported across the system;\n- placing plugin options in an unrelated global configuration model.\n\n## Private input document\n\n`internal/infrastructure/configuration` MAY use a complete decoded document\nwhile discovering files, decoding sections, and applying overlays. It is an\ninput-boundary DTO only.\n\nThe document:\n\n- remains private to the configuration infrastructure;\n- is composed from separately defined feature sections;\n- is immediately translated by application composition;\n- is never returned as the application's runtime model;\n- is never passed through runtime components.\n\nExternal DTOs and domain runtime settings are separate types.\n\nPlugins supply external configuration definitions through a narrow extension\ncontract. The core loader MUST NOT gain a field or recognised filename for every\nplugin.\n\nInvalid configuration input MUST NOT replace active valid settings or terminate\nunaffected capabilities. Configuration changes use the candidate validation,\npromotion, audit, and recovery requirements in\n[safe-change-and-recovery.md](/dev/architecture/safe-change-and-recovery/).\n\n## Dissolution of `internal/config`\n\n`internal/config` has no target location. It MUST be dissolved and MUST NOT\nsurvive as a shared package, configuration domain, or renamed global model.\n\nIts loading behaviour moves to `internal/infrastructure/configuration`. Existing\ntypes and methods are reassigned by behaviour. Repository execution policy,\ndispatch policy, retry policy, ecosystem defaults, tool policy, memory settings,\nprovider settings, and channel settings move to their\nrespective owners.\n\n## Repository settings\n\nThe existing `config.Repo` MUST NOT move intact into a repository domain or\nshared settings package. A managed repository is an application association:\n\n- work intake owns which repository references an identity monitors;\n- task execution owns gates, preflight, test protection, ecosystem behaviour,\n and change limits;\n- task lifecycle owns retry behaviour;\n- scheduling owns concurrency policy;\n- workspace management owns base branch and persistent-storage requirements.\n\nApplication composition correlates these settings by repository reference. No\ndomain receives a universal repository profile.\n\n## Documentation\n\nGenerated configuration reference describes external sections and their owning\ncapabilities. It MUST NOT establish a global Go configuration structure as the\npublic contract.\n\n## Value classification rule\n\nEvery value that controls behaviour MUST have an explicit owner and MUST be\nclassified as exactly one of:\n\n1. **Invariant** — changing it changes the meaning of the program, a wire\n protocol, or a domain concept. It may be a Go constant.\n2. **Runtime setting** — an operator, identity, domain, plugin, or deployment\n may legitimately choose a different value. It is typed configuration owned by\n the affected capability.\n3. **Policy value** — it expresses what is allowed, limited, routed, retried,\n protected, or gated. Its typed definition belongs to the owning domain and it\n is evaluated through the policy system.\n4. **Derived value** — it is computed from other settings and is not\n independently mutable.\n5. **Runtime state** — it changes as the application operates, such as the\n active model for a conversation. It is changed by a domain command and is not\n configuration merely because it can be changed.\n\nAn unclassified behavioural literal is a defect.\n\n`const` MUST NOT be used merely because a value currently has one default.\nConstants are reserved for actual invariants: enum discriminants, protocol\nversions, fixed compatibility tokens, and values whose alteration would change\nthe program's semantics rather than configure an installation.\n\nIn Go, a constant is a compile-time value and does not represent mutable runtime\nstorage. It generally has no address that can be observed or changed.\nArchitecturally, a constant therefore asserts that the value cannot change for\nthe lifetime of the application. Any value that may legitimately change while\nthe application is running, between application instances, or between\ndeployments is not a constant.\n\nA runtime setting may have a default, but the default is not its only access\npath. The owner exposes a typed settings structure, validation, and a\n`DefaultSettings` constructor or equivalent. Private constants used to implement\ndefaults do not turn those defaults into invariants and MUST NOT be consumed\ndirectly by runtime code.\n\nEach runtime setting and policy value MUST declare:\n\n- its owner;\n- type, units, validation constraints, and safe bounds;\n- default and derivation, if any;\n- whether it is startup-only, live-reloadable, or restart-required;\n- whether it may differ by installation, identity, repository, workflow,\n channel, or plugin instance;\n- whether it is secret, sensitive, or safe to report;\n- its current source, effective value, version, and last-known-good version;\n- the command and identity required to change it.\n\n## Current configuration migration ledger\n\nThis ledger records the current meanings hidden inside `internal/config`. It is\nthe source inventory for dissolving that package; rows are split where the\nexisting structure combines different owners.\n\n| Current values | Current location | Target owner and requirement |\n| ----------------------------------------------------------------------------------------- | --------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |\n| `WorkDir`, `StateDir` | root `Config` | Workspace/storage infrastructure settings, bootstrap-only. `StateDir` holds the embedded NATS store and endpoint file, task logs and the readiness disk target. Defaults derive from the platform data directory. |\n| `SkillsDir`, `PluginDir` | root `Config` | Skill and plugin infrastructure settings, independently typed. |\n| `PollInterval` | root `Config` | Work-intake scheduling setting; not a daemon-wide constant. |\n| `Label`, `BotUser`, `BotEmail` | root `Config` | Identity and dispatch associations. `BotEmail` may be derived from the selected forge account but remains overrideable. |\n| `MaxRetries` | root `Config`, `Repo` | Retry policy owned by the operation being retried. The duplicate global/repository representation must disappear. |\n| `DiffCapLines` | root `Config` | Agent-change policy value. |\n| `MaxSteps`, `WallClock`, `GateMaxFailures` | `Budgets` | Workflow-execution policy, with units and zero/unlimited semantics made explicit. |\n| `Class`, `APIKeyEnv`, `BaseURL` | `Provider` | Provider instance settings. Secret lookup becomes a `SecretRef`, not an environment variable name leaking into the provider domain. |\n| `Mode`, `Command`, `Env` | `LegacyAgent` | Decode-only compatibility input for the removed executor selector. Delete after the config migration window; it has no runtime settings owner. |\n| `Type`, `Host`, `Token` / `TokenEnv` (older form) | `Forge` | Forge adapter settings. The older `TokenEnv` input is accepted only at the loader boundary. |\n| `Trigger`, `AckReaction`, label names | `Dispatch` | Work-intake and forge presentation policy. Label vocabulary is configurable per forge/identity association. |\n| `Engine` | `MemoryConfig` | Memory engine selection. `internal/domain/memory` is the only engine family; per-engine settings remain owned and validated by that engine. |\n| `Name`, `Transport`, `Command`, `Args`, `WorkDir`, `URL` | `MCPServer` | One typed MCP server instance specification. Transport-specific variants must reject fields belonging to other variants. |\n| `MaxResultChars` | `ToolPolicy` | Tool-execution policy values, not generic daemon configuration. |\n| `URL`, `TokenEnv` | `NATSConfig` | NATS transport settings with a secret reference replacing the environment-name field. |\n| `Image`, `MaxConcurrency`, `MaxUptime`, `VolumeTTL`, `PullPolicy`, `Network` | `ContainerConfig` | Mandatory managed-worker infrastructure settings and resource policy. The legacy `enabled` input is decode-only. |\n| model mappings, email, webhook, Telegram | `ChatConfig` | Separate channel instance settings plus channel-routing associations; there is no chat-wide settings owner. |\n| `AllowedUserIDs`, `Token` / `TokenEnv` (older form), update commands | Telegram config | Telegram adapter settings and access policy. `Token` is the preferred `secret.SecretRef`; `TokenEnv` remains a backwards-compatible fallback. |\n| listen/relay addresses | email config | Email adapter instance settings. |\n| webhook target | `Notify` | Notification adapter instance settings. |\n| web listen address | `Web` | Web transport settings. |\n| `listen` under `[services.state]` and `[services.gateway]` | `ServiceConnection` | The bind address of the process that owns the service. Read only by that process, and defaulted to the address its clients dial so the pair cannot drift. |\n| forge, dispatch, models, providers, containers, memory, tools and repositories | `IdentityConfig` | Existing composite duplicates unrelated settings. Replace it with associations from an `IdentityID` to independently owned configured instances. |\n| owner, name, base, gate, protection, ecosystem, preflight, storage, retry and concurrency | `Repo` | Split across work intake, workspace, workflow execution, policy, gate, and scheduling as described in Repository settings. |\n| `TaskConfig`, `TaskForge` | worker snapshot | Replace with the minimum immutable, versioned WorkflowExecution input assembled by the application. It is not a second configuration model. |\n| `Extra` and arbitrary maps | root/provider config | Allowed only at an extension boundary whose owning plugin/provider supplies schema and validation. Core configuration never consumes an untyped catch-all. |\n\n## Behavioural literals and incorrectly fixed values\n\nThe following current values affect useful behaviour but are not represented as\nowned typed settings or policy. They MUST be classified and migrated before the\nold configuration package is removed. The listed target is the architectural\ndecision; exact setting names are implementation details.\n\n| Capability | Values fixed in current code | Required classification |\n| ---------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |\n| Chat agent | maximum steps `8` in `cmd/archied` | Workflow/chat execution policy. |\n| Context compression | enabled, threshold `0.5`, context size `128000`, protect first `3` and last `20`, compression marker, token estimate of four characters per token | Model/context runtime settings. The marker is presentation; the estimate is provider/model strategy, not a universal truth. |\n| Tool guardrails | escalation thresholds `3`, `5`, `10`, `2` | Tool-use policy with named meanings and bounded defaults. |\n| Tool output | per-result cap `50000` characters, no aggregate turn cap, preview `500` characters | Tool-execution resource and presentation settings. Per-result spill/truncation remains; turn continuation is not character- or token-gated. |\n| Pairing/access | code TTL `1h`, maximum pending `3`, rate window `10m`, failures `5`, lockout `1h`, maximum lockout `24h` | Access policy. Code length/alphabet and salt length require an explicit security-constraint decision; they are not automatically valid constants. |\n| NATS daemon consumer | stream `ARCHIE_TASKS`, consumer `archie-daemon`, deduplication `2m`, poll `2s`, deliveries `3`, acknowledgement wait `5m`, inactive threshold `24h`, subjects | NATS topology settings plus delivery policy. Subject grammar or protocol headers may be invariants only if declared as versioned wire contracts. |\n| NATS worker consumer | stream `ARCHIE_TASKS`, consumer `archie-agent`, deduplication `2m`, poll `5s`, acknowledgement wait `30m`, deliveries `3`, inactive threshold `24h`, queue group and retry delay `1s` | Worker transport settings and delivery/retry policy. Shared stream defaults must have one owner rather than two copies. |\n| Worker readiness | readiness timeout `20s`, retry delay `250ms` | Worker lifecycle and retry policy. |\n| Execution/RPC | agent execution `30m`, task-run RPC `60s`, worktree RPC `15m` | Per-operation timeout policy. |\n| Plugin/tool cleanup | plugin rollback `10s`, tool-provider cleanup `10s`, MCP-provider cleanup `10s` | Lifecycle shutdown/rollback policy owned by each engine, with a shared policy vocabulary where appropriate. |\n| Memory read path | per-scope record limit `20`, rendered `\u003cmemory\u003e` block byte cap `8192` | Memory read policy owned by the engine family (`internal/domain/memory`). |\n| Built-in memory store | per-file size limit `100KiB`, query result limit `100` | Split into memory resource policy, input policy, and presentation settings. |\n| Memory safety scanner | compiled prompt-injection and sensitive-data patterns with fixed warn/block consequences | Versioned memory-domain policy definitions. Patterns and consequences must be inspectable and replaceable without editing scanner code. |\n| MCP stdio | initial backoff `500ms`, maximum backoff `30s`, unlimited default retries, send and shutdown `5s` | MCP transport retry, request, and lifecycle policy. Unlimited retries must be explicit rather than a zero-value accident. |\n| MCP SSE/HTTP | reconnect `1s` to `30s`, POST/request timeout `30s` | MCP transport settings and retry policy. |\n| Message framing | MCP frame `4MiB`, worker protocol frame `8MiB`, diagnostic `64KiB`, media/base64 `64MiB` | Resource/security policy unless a published wire protocol mandates the limit. Limits must be configurable only within safe bounds. |\n| Webhooks and web UI | webhook bind `0.0.0.0:8644`, webhook shutdown `5s`, web read-header `5s`, web shutdown `2s` | Transport instance and lifecycle settings. |\n| Telegram | typing refresh `4s`, draft interval `700ms`, update-action TTL `10m` | Telegram adapter presentation and lifecycle settings. |\n| Telegram limits | message length `4000` and callback/update prefixes | Record the upstream platform constraint and version it as an adapter compatibility limit; prefixes may remain wire constants when changing them would invalidate outstanding callbacks. |\n| Containers | stop and pull timeouts `10s` and `30s` | Container lifecycle policy. |\n| HTTP clients | feasibility notification timeout `10s`; other paths use `http.DefaultClient` | Adapter request policy. Every outbound client requires an explicit timeout profile. |\n| Workflow routing | labels `bug`, `feature`, `bootstrap`; workflows `tdd`, `feasibility`, `bootstrap`; fallback `implement` | Agent-system routing policy. Workflow identifiers are references, not compiled routing decisions. |\n| Filesystem/deployment | `.agents/skills`, `.archie/stages`, `.archie/gate.go`, `/data/worktree`, `/data/repo`, cache names/paths and prepared sentinel | Split into deployment settings, repository conventions, and genuine internal protocol markers. Paths an installation may relocate are settings. |\n| Packaged release data | `/usr/share/archie/CHANGELOG.md` | Packaging/deployment settings or paths derived from an installation prefix. |\n| Forge presentation | default label colour `bfd4f2` and state-label colours | Forge adapter presentation settings. |\n| Fallback system prompt | compiled fallback prompt | Agent-system default content, versioned and replaceable through its owning settings; safety constraints remain policy/code. |\n| Default paths and identities | config path, data paths, listen addresses, model roles and workflow names | Typed defaults belonging to their respective owners, never global constants consumed throughout the application. |\n\nThis inventory is intentionally broader than exported Go `const` declarations. A\nliteral hidden in a constructor, timeout, switch, retry loop, path join, prompt,\nlabel map, or client setup restricts behaviour just as effectively as an\nincorrectly used constant.\n\n## Constants that may remain\n\nA value may remain constant only when its owner documents why runtime variation\nwould be invalid. Typical valid cases are:\n\n- domain enum values and state names;\n- a versioned wire-protocol version or framing token;\n- reserved internal headers and callback prefixes that peers must interpret\n identically;\n- compile-time mathematical relationships;\n- sentinel errors and internal type identifiers.\n\nNames such as a stream, consumer, label, workflow, file, directory, network,\nmodel role, timeout, retry count, capacity, size limit, colour, prompt, command,\nor executable are not invariants merely because the current application has only\none of them.\n\nArchitecture tests SHOULD reject exported behavioural constants outside approved\ninvariant packages and SHOULD detect direct consumption of private default\nconstants outside their owning `DefaultSettings` construction.\n\n## Current mutation and reload ledger\n\nThe application currently changes behavioural values through unrelated paths:\n\n| Current change seam | Current behaviour | Required target |\n| ---------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |\n| Telegram `/restart` in `cmd/archied` | Re-reads the global document and directly replaces Telegram token and allowed-user IDs; failure retains old values. | Submit a Telegram-settings candidate through the universal change protocol. Validate the complete candidate, promote atomically, audit the acting identity, and retain last-known-good. |\n| Active model selection | `chatModelManager.SetActiveModel` changes a model reference. | A conversation/session command changing runtime state. Validate against configured model instances and audit it; do not rewrite configuration. |\n| Active provider selection | `chatModelManager.SetActiveProvider` changes provider selection. | A conversation/session command changing runtime state, with the same distinction from provider-instance configuration. |\n| Persona selection | `PersonaRegistry.SetActive` mutates an in-memory per-session choice. | Agent-system session state. Persistence and audit requirements are owned by that domain, not the configuration loader. |\n| File and environment overlays | Applied while loading `internal/config`; defaults are filled by mutation in `finalize`. | Configuration-infrastructure input processing that produces separately typed candidates. Defaults and derivations remain visible in provenance. |\n| SIGHUP config reload (2026-08) | Re-resolves the file config, re-applies the database layer over it, validates, then atomically republishes through `config.Holder`. A failed reload, whether the file or the control plane caused it, keeps the running config and records `last_error`/`last_error_at` in `/api/config`. | Candidate promotion with last-known-good retention: the running snapshot is untouched on failure. A bounded observation period and actor audit are not yet implemented. |\n| Control-plane resource command (2026-09) | `ControlPlaneService.Command` validates a whole resource document against its owning feature, then writes it with an audit record (actor, source, request ID, expected version) under optimistic concurrency. Hosted on the State Store's gRPC server; the Web UI and messaging channels are its only clients. See `docs/prds/runtime-control-plane.md`. | Met for validate-stage-promote, actor audit and versioned history. Per-field health observation and validate-before-switch on live apply are not yet implemented. |\n\nAny further mutation seam discovered during migration MUST be added here before\nit is changed. Direct field assignment on a live settings object is prohibited.\n\n## Universal change protocol\n\nChanging configuration is an application command performed by an identity. It\nMUST:\n\n1. resolve the target owner and current version;\n2. parse into that owner's typed settings;\n3. validate syntax, semantic constraints, policy, references, and safe bounds;\n4. stage a candidate without altering the active value;\n5. run owner-defined readiness and health checks;\n6. atomically promote the candidate and record actor, source, reason, diff,\n prior version, and new version;\n7. observe the affected capability for a bounded period;\n8. retain the new version on success, or automatically restore last-known-good\n on failure;\n9. report the rejected or rolled-back candidate without terminating unaffected\n capabilities.\n\nStartup loading uses the same candidate rules. A bad user value therefore does\nnot partially mutate the application or make a previously healthy capability\nunavailable.\n\nSettings that cannot safely reload in-process are marked restart-required. The\nchange may still be validated and versioned immediately, but promotion occurs\nthrough a controlled restart with the same health and rollback semantics.\n\n## Implemented runtime reload and settings writes (2026-09)\n\nTwo mechanisms change behavioural values at runtime: SIGHUP re-reads the file\nconfig, and the control plane owns every database-backed setting. The dashboard\nruntime overlay that used to sit between them was deleted once the control plane\ncovered its surface; `-config-overlay` names a file overlay and is unrelated.\n\n### Published snapshot: `config.Holder`\n\nThe running configuration is a published snapshot held in `config.Holder`\n(`internal/config/holder.go`). Get returns a value under RLock; Set replaces the\nwhole snapshot. A published Config is immutable: reload constructs a fresh one\nand calls Set. `config.Clone` deep-copies every reference-type field so a failed\ndecode cannot mutate the published snapshot's shared maps. `Holder.Get` panics\non a nil receiver — a forgotten `NewHolder` must fail loudly, not boot a daemon\nthat silently runs on an empty config.\n\n### The reloadable-field criterion\n\nThe diff-and-warn on reload asks whether a changed field takes effect without a\nrestart. The criterion is: **a field is reloadable when every consumer re-reads\nit.** That question read as \"what does the daemon re-read\" while the daemon owned\nthe only Holder; it stopped being sufficient when webui began sharing that\nHolder, because webui handlers re-read on every request. The allowlist lives in\n`cmd/archied/reload.go` and is deliberately a deny-by-default list: a field added\nto Config later defaults to requires-restart, forcing whoever adds it to decide\ndeliberately. A reload that changes any non-allowlisted field logs\n\"requires a restart\" with the field names; a fully-applied reload logs Info.\n\n### SIGHUP reload\n\nSIGHUP re-resolves the file config, re-applies the database layer over it\n(`boot.runtimeConfig`, the same call boot makes), validates, and atomically\nrepublishes through the shared Holder. A failed reload keeps the running config\nand records `last_error`/`last_error_at` in `/api/config` so the stale state is\nvisible where the operator is looking. The three poll tickers re-read\n`PollInterval` per iteration and reset on change, so a reloaded interval\ngenuinely takes effect.\n\n### Settings writes go through the control plane\n\nDatabase-backed settings are read and written through\n`ControlPlaneService` on the State Store's gRPC server: catalogue, query,\ncommand, watch. Each feature owns its resource's validation and commands;\nstorage writes versions and audit records only. `docs/prds/runtime-control-plane.md`\nis the specification.\n\nTwo properties matter to anyone changing this area:\n\n- **Bootstrap stays file-owned.** The daemon reads the file config first and\n layers database resources over it at boot (`boot.loadRuntimeConfig`). Values\n the daemon needs before it can reach the State Store — the database path,\n listen addresses, credentials — stay file-only by design.\n- **Reload re-reads them.** SIGHUP resolves the file document and then runs the\n same `boot.runtimeConfig` layering boot does, so a reload cannot revert a\n database-owned setting to its file value. A control plane that cannot answer\n fails the reload rather than publishing the file's values over the running\n ones. Workflow execution budgets arrive on a watch rather than in a resource\n query, so boot records the last published settings and the reload re-applies\n them from there.\n- **Live kinds arrive on a watch.** Provider settings, model role assignments,\n repository policies, scheduling policy, plugin settings, review settings and\n channel settings are `ApplyMode: live`: the daemon keeps a watch per kind and,\n on a new version, re-runs `boot.runtimeConfig` over the published snapshot and\n republishes through `config.Holder`. A candidate the layered document cannot\n run is refused before it is published — last-known-good keeps running and\n the refusal is reported through apply status. A kind whose consumer lives in\n another process reconciles there too: `archie-messaging` re-reads the stored\n channel settings on the apply-status restamp interval and restarts only the\n channel whose transport settings changed, while the listen addresses it has\n bound and the set of channels it composed stay process bindings that take\n effect on restart. A kind joins the live list only with a consumer that\n re-reads it; one whose startup-built consumer never does stays\n restart-required, and the model/provider half of the live set also rebuilds\n the gateway chat runtime's provider set and the chat model list the chat\n surfaces offer.\n\n## Completion criteria\n\nThe configuration rearchitecture is complete only when:\n\n- `internal/config` no longer exists;\n- every behavioural value has one classification and owner;\n- every current field and hard-coded value in the ledgers has a recorded\n disposition;\n- domains and plugins receive only their typed settings;\n- all changes use semantic commands and the universal change protocol;\n- active, candidate, rejected, rolled-back, and last-known-good versions are\n queryable with secret-safe provenance;\n- generated documentation lists every external key, default, constraint,\n mutability class, owner, and deprecation;\n- architecture tests prevent a new central runtime configuration model, direct\n live-setting mutation, and incorrectly fixed behavioural values.\n\n## Startup policy: missing credential degrades, invalid config is fatal\n\nDecided 2026-07-30. Recovered 2026-08-09 from the pre-migration issue tracker.\n\nA missing **credential** disables a capability. Only an **invalid config** stops\nthe daemon. \"Forge is a feature, not a requirement.\"\n\nBefore `518a1a0`, an unresolvable forge token did `return 1` in\n`cmd/archied/main.go`, so a chat-only user who never configured GitHub got no\nchat, no gateway, nothing — and under a systemd unit with `Restart=on-failure`\nand no start limit it crash-looped every five seconds with the error scrolling\npast unread. Forge construction now lives in `resolveForge`, shared by the\nprimary forge and every configured identity, and degrades to `forge.NewNoop`\nwith a warning. The identity path had the same defect and was worse: one\nidentity's missing credential killed all the others.\n\nMalformed configuration still fails fast in `configuration.Validate`, well\nbefore this point. That distinction is the whole design.\n\nApply the same reasoning to any future capability credential, LLM keys\nespecially: start, report the capability as unavailable, let the operator see it.\nGuarded by `TestResolveForgeDegradesInsteadOfFailing`.\n\n**Known consequence, tracked separately:** `worktree.Manager` still receives the\nempty token, so pushes fail late rather than at startup.\n\n## Config generation belongs to `archied setup`, not `install.sh`\n\nDecided 2026-07-30 ([#368](https://github.com/samcharles93/archie-core/issues/368)).\nRecovered 2026-08-09 from the pre-migration issue tracker.\n\n`install.sh` must stop generating `config.toml`; `archied setup` owns it. The\ninstaller hand-rolling TOML has already drifted from the schema once: the\ngenerated-config branch hardcoded `key = ARCHIE_GITHUB_TOKEN` and\n`host = ${GITEA_URL}` regardless of the answers given, so choosing Gitea produced\na config naming a token the installer never wrote (archied then exited 1), and\nchoosing GitHub pointed `host` at `gitea.example.com`. `61a5b35` patched this\nwith a `forge_block` shell helper shared by both config paths — that helper is a\n**workaround**, deleted by\n[#72](https://github.com/samcharles93/archie-core/issues/72). The durable fix is\nthat the code writing the config is the code that reads it.\n\nWhy Go and not Python: the installer must run where nothing is installed yet, and\n`python3` with `tomllib` means 3.11+, which is not safe on LTS distros. `archied`\nis already being built two steps earlier.\n\n**Key constraint for the port:** `BurntSushi/toml` does **not** survive a\ndecode/encode round trip with comments intact, and `config.example.toml` carries\ndocumentation users are meant to hand-edit. Decide the write strategy\n([#412](https://github.com/samcharles93/archie-core/issues/412)) before writing\nany prompting code. Do not port a separate `config.yaml`/`auth.yaml` secret\nsplit — Archie already has `secret.SecretRef{engine,key}`, so setup writes the\nreference to TOML and the value to the secret engine.\n" + "body": "# Configuration\n\n**Status:** Approved; existing-type assignment is ongoing \n**Date:** 2026-07-28 \n**Tracking issue:** [#73](https://github.com/samcharles93/archie-core/issues/73)\n\n## Ownership\n\nConfiguration has two separate responsibilities:\n\n1. `internal/infrastructure/configuration` loads files, environment variables,\n secret references, overlays, and compatibility representations.\n2. Each domain and plugin owns the typed runtime settings required for its\n behaviour.\n\n`internal/app` translates external input into those settings, validates\ncross-capability combinations, and supplies each component only its own\nsettings.\n\nThe following are prohibited:\n\n- passing complete application configuration into a domain;\n- making a domain depend on TOML, YAML, environment-variable names, secret\n references, or a configuration library;\n- defining a central runtime configuration type imported across the system;\n- placing plugin options in an unrelated global configuration model.\n\n## Private input document\n\n`internal/infrastructure/configuration` MAY use a complete decoded document\nwhile discovering files, decoding sections, and applying overlays. It is an\ninput-boundary DTO only.\n\nThe document:\n\n- remains private to the configuration infrastructure;\n- is composed from separately defined feature sections;\n- is immediately translated by application composition;\n- is never returned as the application's runtime model;\n- is never passed through runtime components.\n\nExternal DTOs and domain runtime settings are separate types.\n\nPlugins supply external configuration definitions through a narrow extension\ncontract. The core loader MUST NOT gain a field or recognised filename for every\nplugin.\n\nInvalid configuration input MUST NOT replace active valid settings or terminate\nunaffected capabilities. Configuration changes use the candidate validation,\npromotion, audit, and recovery requirements in\n[safe-change-and-recovery.md](/dev/architecture/safe-change-and-recovery/).\n\n## Dissolution of `internal/config`\n\n`internal/config` has no target location. It MUST be dissolved and MUST NOT\nsurvive as a shared package, configuration domain, or renamed global model.\n\nIts loading behaviour moves to `internal/infrastructure/configuration`. Existing\ntypes and methods are reassigned by behaviour. Repository execution policy,\ndispatch policy, retry policy, ecosystem defaults, tool policy, memory settings,\nprovider settings, and channel settings move to their\nrespective owners.\n\n## Repository settings\n\nThe existing `config.Repo` MUST NOT move intact into a repository domain or\nshared settings package. A managed repository is an application association:\n\n- work intake owns which repository references an identity monitors;\n- task execution owns gates, preflight, test protection, ecosystem behaviour,\n and change limits;\n- task lifecycle owns retry behaviour;\n- scheduling owns concurrency policy;\n- workspace management owns base branch and persistent-storage requirements.\n\nApplication composition correlates these settings by repository reference. No\ndomain receives a universal repository profile.\n\n## Documentation\n\nGenerated configuration reference describes external sections and their owning\ncapabilities. It MUST NOT establish a global Go configuration structure as the\npublic contract.\n\n## Value classification rule\n\nEvery value that controls behaviour MUST have an explicit owner and MUST be\nclassified as exactly one of:\n\n1. **Invariant** — changing it changes the meaning of the program, a wire\n protocol, or a domain concept. It may be a Go constant.\n2. **Runtime setting** — an operator, identity, domain, plugin, or deployment\n may legitimately choose a different value. It is typed configuration owned by\n the affected capability.\n3. **Policy value** — it expresses what is allowed, limited, routed, retried,\n protected, or gated. Its typed definition belongs to the owning domain and it\n is evaluated through the policy system.\n4. **Derived value** — it is computed from other settings and is not\n independently mutable.\n5. **Runtime state** — it changes as the application operates, such as the\n active model for a conversation. It is changed by a domain command and is not\n configuration merely because it can be changed.\n\nAn unclassified behavioural literal is a defect.\n\n`const` MUST NOT be used merely because a value currently has one default.\nConstants are reserved for actual invariants: enum discriminants, protocol\nversions, fixed compatibility tokens, and values whose alteration would change\nthe program's semantics rather than configure an installation.\n\nIn Go, a constant is a compile-time value and does not represent mutable runtime\nstorage. It generally has no address that can be observed or changed.\nArchitecturally, a constant therefore asserts that the value cannot change for\nthe lifetime of the application. Any value that may legitimately change while\nthe application is running, between application instances, or between\ndeployments is not a constant.\n\nA runtime setting may have a default, but the default is not its only access\npath. The owner exposes a typed settings structure, validation, and a\n`DefaultSettings` constructor or equivalent. Private constants used to implement\ndefaults do not turn those defaults into invariants and MUST NOT be consumed\ndirectly by runtime code.\n\nEach runtime setting and policy value MUST declare:\n\n- its owner;\n- type, units, validation constraints, and safe bounds;\n- default and derivation, if any;\n- whether it is startup-only, live-reloadable, or restart-required;\n- whether it may differ by installation, identity, repository, workflow,\n channel, or plugin instance;\n- whether it is secret, sensitive, or safe to report;\n- its current source, effective value, version, and last-known-good version;\n- the command and identity required to change it.\n\n## Current configuration migration ledger\n\nThis ledger records the current meanings hidden inside `internal/config`. It is\nthe source inventory for dissolving that package; rows are split where the\nexisting structure combines different owners.\n\n| Current values | Current location | Target owner and requirement |\n| ----------------------------------------------------------------------------------------- | --------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |\n| `WorkDir`, `StateDir` | root `Config` | Workspace/storage infrastructure settings, bootstrap-only. `StateDir` holds the embedded NATS store and endpoint file, task logs and the readiness disk target. Defaults derive from the platform data directory. |\n| `SkillsDir`, `PluginDir` | root `Config` | Skill and plugin infrastructure settings, independently typed. |\n| `PollInterval` | root `Config` | Work-intake scheduling setting; not a daemon-wide constant. |\n| `Label`, `BotUser`, `BotEmail` | root `Config` | Identity and dispatch associations. `BotEmail` may be derived from the selected forge account but remains overrideable. |\n| `MaxRetries` | root `Config`, `Repo` | Retry policy owned by the operation being retried. The duplicate global/repository representation must disappear. |\n| `DiffCapLines` | root `Config` | Agent-change policy value. |\n| `MaxSteps`, `WallClock`, `GateMaxFailures` | `Budgets` | Workflow-execution policy, with units and zero/unlimited semantics made explicit. |\n| `Class`, `APIKeyEnv`, `BaseURL` | `Provider` | Provider instance settings. Secret lookup becomes a `SecretRef`, not an environment variable name leaking into the provider domain. |\n| `Mode`, `Command`, `Env` | `LegacyAgent` | Decode-only compatibility input for the removed executor selector. Delete after the config migration window; it has no runtime settings owner. |\n| `Type`, `Host`, `Token` / `TokenEnv` (older form) | `Forge` | Forge adapter settings. The older `TokenEnv` input is accepted only at the loader boundary. |\n| `Trigger`, `AckReaction`, label names | `Dispatch` | Work-intake and forge presentation policy. Label vocabulary is configurable per forge/identity association. |\n| `Engine` | `MemoryConfig` | Memory engine selection. `internal/domain/memory` is the only engine family; per-engine settings remain owned and validated by that engine. |\n| `Name`, `Transport`, `Command`, `Args`, `WorkDir`, `URL` | `MCPServer` | One typed MCP server instance specification. Transport-specific variants must reject fields belonging to other variants. |\n| `MaxResultChars` | `ToolPolicy` | Tool-execution policy values, not generic daemon configuration. |\n| `URL`, `TokenEnv` | `NATSConfig` | NATS transport settings with a secret reference replacing the environment-name field. |\n| `Image`, `MaxConcurrency`, `MaxUptime`, `VolumeTTL`, `PullPolicy`, `Network` | `ContainerConfig` | Mandatory managed-worker infrastructure settings and resource policy. The legacy `enabled` input is decode-only. |\n| model mappings, email, webhook, Telegram | `ChatConfig` | Separate channel instance settings plus channel-routing associations; there is no chat-wide settings owner. |\n| `AllowedUserIDs`, `Token` / `TokenEnv` (older form), update commands | Telegram config | Telegram adapter settings and access policy. `Token` is the preferred `secret.SecretRef`; `TokenEnv` remains a backwards-compatible fallback. |\n| listen/relay addresses | email config | Email adapter instance settings. |\n| webhook target | `Notify` | Notification adapter instance settings. |\n| web listen address | `Web` | Web transport settings. |\n| `listen` under `[services.state]` and `[services.gateway]` | `ServiceConnection` | The bind address of the process that owns the service. Read only by that process, and defaulted to the address its clients dial so the pair cannot drift. |\n| forge, dispatch, models, providers, containers, memory, tools and repositories | `IdentityConfig` | Existing composite duplicates unrelated settings. Replace it with associations from an `IdentityID` to independently owned configured instances. |\n| owner, name, base, gate, protection, ecosystem, preflight, storage, retry and concurrency | `Repo` | Split across work intake, workspace, workflow execution, policy, gate, and scheduling as described in Repository settings. |\n| `TaskConfig`, `TaskForge` | worker snapshot | Replace with the minimum immutable, versioned WorkflowExecution input assembled by the application. It is not a second configuration model. |\n| `Extra` and arbitrary maps | root/provider config | Allowed only at an extension boundary whose owning plugin/provider supplies schema and validation. Core configuration never consumes an untyped catch-all. |\n\n## Behavioural literals and incorrectly fixed values\n\nThe following current values affect useful behaviour but are not represented as\nowned typed settings or policy. They MUST be classified and migrated before the\nold configuration package is removed. The listed target is the architectural\ndecision; exact setting names are implementation details.\n\n| Capability | Values fixed in current code | Required classification |\n| ---------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |\n| Chat agent | maximum steps `8` in `cmd/archied` | Workflow/chat execution policy. |\n| Context compression | enabled, threshold `0.5`, context size `128000`, protect first `3` and last `20`, compression marker, token estimate of four characters per token | Model/context runtime settings. The marker is presentation; the estimate is provider/model strategy, not a universal truth. |\n| Tool guardrails | escalation thresholds `3`, `5`, `10`, `2` | Tool-use policy with named meanings and bounded defaults. |\n| Tool output | per-result cap `50000` characters, no aggregate turn cap, preview `500` characters | Tool-execution resource and presentation settings. Per-result spill/truncation remains; turn continuation is not character- or token-gated. |\n| Pairing/access | code TTL `1h`, maximum pending `3`, rate window `10m`, failures `5`, lockout `1h`, maximum lockout `24h` | Access policy. Code length/alphabet and salt length require an explicit security-constraint decision; they are not automatically valid constants. |\n| NATS daemon consumer | stream `ARCHIE_TASKS`, consumer `archie-daemon`, deduplication `2m`, poll `2s`, deliveries `3`, acknowledgement wait `5m`, inactive threshold `24h`, subjects | NATS topology settings plus delivery policy. Subject grammar or protocol headers may be invariants only if declared as versioned wire contracts. |\n| NATS worker consumer | stream `ARCHIE_TASKS`, consumer `archie-agent`, deduplication `2m`, poll `5s`, acknowledgement wait `30m`, deliveries `3`, inactive threshold `24h`, queue group and retry delay `1s` | Worker transport settings and delivery/retry policy. Shared stream defaults must have one owner rather than two copies. |\n| Worker readiness | readiness timeout `20s`, retry delay `250ms` | Worker lifecycle and retry policy. |\n| Execution/RPC | agent execution `30m`, task-run RPC `60s`, worktree RPC `15m` | Per-operation timeout policy. |\n| Plugin/tool cleanup | plugin rollback `10s`, tool-provider cleanup `10s`, MCP-provider cleanup `10s` | Lifecycle shutdown/rollback policy owned by each engine, with a shared policy vocabulary where appropriate. |\n| Memory read path | per-scope record limit `20`, rendered `\u003cmemory\u003e` block byte cap `8192` | Memory read policy owned by the engine family (`internal/domain/memory`). |\n| Built-in memory store | per-file size limit `100KiB`, query result limit `100` | Split into memory resource policy, input policy, and presentation settings. |\n| Memory safety scanner | compiled prompt-injection and sensitive-data patterns with fixed warn/block consequences | Versioned memory-domain policy definitions. Patterns and consequences must be inspectable and replaceable without editing scanner code. |\n| MCP stdio | initial backoff `500ms`, maximum backoff `30s`, unlimited default retries, send and shutdown `5s` | MCP transport retry, request, and lifecycle policy. Unlimited retries must be explicit rather than a zero-value accident. |\n| MCP SSE/HTTP | reconnect `1s` to `30s`, POST/request timeout `30s` | MCP transport settings and retry policy. |\n| Message framing | MCP frame `4MiB`, worker protocol frame `8MiB`, diagnostic `64KiB`, media/base64 `64MiB` | Resource/security policy unless a published wire protocol mandates the limit. Limits must be configurable only within safe bounds. |\n| Webhooks and web UI | webhook bind `0.0.0.0:8644`, webhook shutdown `5s`, web read-header `5s`, web shutdown `2s` | Transport instance and lifecycle settings. |\n| Telegram | typing refresh `4s`, draft interval `700ms`, update-action TTL `10m` | Telegram adapter presentation and lifecycle settings. |\n| Telegram limits | message length `4000` and callback/update prefixes | Record the upstream platform constraint and version it as an adapter compatibility limit; prefixes may remain wire constants when changing them would invalidate outstanding callbacks. |\n| Containers | stop and pull timeouts `10s` and `30s` | Container lifecycle policy. |\n| HTTP clients | feasibility notification timeout `10s`; other paths use `http.DefaultClient` | Adapter request policy. Every outbound client requires an explicit timeout profile. |\n| Workflow routing | labels `bug`, `feature`, `bootstrap`; workflows `tdd`, `feasibility`, `bootstrap`; fallback `implement` | Agent-system routing policy. Workflow identifiers are references, not compiled routing decisions. |\n| Filesystem/deployment | `.agents/skills`, `.archie/stages`, `.archie/gate.go`, `/data/worktree`, `/data/repo`, cache names/paths and prepared sentinel | Split into deployment settings, repository conventions, and genuine internal protocol markers. Paths an installation may relocate are settings. |\n| Packaged release data | `/usr/share/archie/CHANGELOG.md` | Packaging/deployment settings or paths derived from an installation prefix. |\n| Forge presentation | default label colour `bfd4f2` and state-label colours | Forge adapter presentation settings. |\n| Fallback system prompt | compiled fallback prompt | Agent-system default content, versioned and replaceable through its owning settings; safety constraints remain policy/code. |\n| Default paths and identities | config path, data paths, listen addresses, model roles and workflow names | Typed defaults belonging to their respective owners, never global constants consumed throughout the application. |\n\nThis inventory is intentionally broader than exported Go `const` declarations. A\nliteral hidden in a constructor, timeout, switch, retry loop, path join, prompt,\nlabel map, or client setup restricts behaviour just as effectively as an\nincorrectly used constant.\n\n## Constants that may remain\n\nA value may remain constant only when its owner documents why runtime variation\nwould be invalid. Typical valid cases are:\n\n- domain enum values and state names;\n- a versioned wire-protocol version or framing token;\n- reserved internal headers and callback prefixes that peers must interpret\n identically;\n- compile-time mathematical relationships;\n- sentinel errors and internal type identifiers.\n\nNames such as a stream, consumer, label, workflow, file, directory, network,\nmodel role, timeout, retry count, capacity, size limit, colour, prompt, command,\nor executable are not invariants merely because the current application has only\none of them.\n\nArchitecture tests SHOULD reject exported behavioural constants outside approved\ninvariant packages and SHOULD detect direct consumption of private default\nconstants outside their owning `DefaultSettings` construction.\n\n## Current mutation and reload ledger\n\nThe application currently changes behavioural values through unrelated paths:\n\n| Current change seam | Current behaviour | Required target |\n| ---------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |\n| Telegram `/restart` in `cmd/archied` | Re-reads the global document and directly replaces Telegram token and allowed-user IDs; failure retains old values. | Submit a Telegram-settings candidate through the universal change protocol. Validate the complete candidate, promote atomically, audit the acting identity, and retain last-known-good. |\n| Active model selection | `chatModelManager.SetActiveModel` changes a model reference. | A conversation/session command changing runtime state. Validate against configured model instances and audit it; do not rewrite configuration. |\n| Active provider selection | `chatModelManager.SetActiveProvider` changes provider selection. | A conversation/session command changing runtime state, with the same distinction from provider-instance configuration. |\n| Persona selection | `PersonaRegistry.SetActive` mutates an in-memory per-session choice. | Agent-system session state. Persistence and audit requirements are owned by that domain, not the configuration loader. |\n| File and environment overlays | Applied while loading `internal/config`; defaults are filled by mutation in `finalize`. | Configuration-infrastructure input processing that produces separately typed candidates. Defaults and derivations remain visible in provenance. |\n| SIGHUP config reload (2026-08) | Re-resolves the file config, re-applies the database layer over it, validates, then atomically republishes through `config.Holder`. A failed reload, whether the file or the control plane caused it, keeps the running config and records `last_error`/`last_error_at` in `/api/config`. | Candidate promotion with last-known-good retention: the running snapshot is untouched on failure. A bounded observation period and actor audit are not yet implemented. |\n| Control-plane resource command (2026-09) | `ControlPlaneService.Command` validates a whole resource document against its owning feature, then writes it with an audit record (actor, source, request ID, expected version) under optimistic concurrency. Hosted on the State Store's gRPC server; the Web UI and messaging channels are its only clients. See `docs/prds/runtime-control-plane.md`. | Met for validate-stage-promote, actor audit and versioned history. Per-field health observation and validate-before-switch on live apply are not yet implemented. |\n\nAny further mutation seam discovered during migration MUST be added here before\nit is changed. Direct field assignment on a live settings object is prohibited.\n\n## Universal change protocol\n\nChanging configuration is an application command performed by an identity. It\nMUST:\n\n1. resolve the target owner and current version;\n2. parse into that owner's typed settings;\n3. validate syntax, semantic constraints, policy, references, and safe bounds;\n4. stage a candidate without altering the active value;\n5. run owner-defined readiness and health checks;\n6. atomically promote the candidate and record actor, source, reason, diff,\n prior version, and new version;\n7. observe the affected capability for a bounded period;\n8. retain the new version on success, or automatically restore last-known-good\n on failure;\n9. report the rejected or rolled-back candidate without terminating unaffected\n capabilities.\n\nStartup loading uses the same candidate rules. A bad user value therefore does\nnot partially mutate the application or make a previously healthy capability\nunavailable.\n\nSettings that cannot safely reload in-process are marked restart-required. The\nchange may still be validated and versioned immediately, but promotion occurs\nthrough a controlled restart with the same health and rollback semantics.\n\n## Implemented runtime reload and settings writes (2026-09)\n\nTwo mechanisms change behavioural values at runtime: SIGHUP re-reads the file\nconfig, and the control plane owns every database-backed setting. The dashboard\nruntime overlay that used to sit between them was deleted once the control plane\ncovered its surface; `-config-overlay` names a file overlay and is unrelated.\n\n### Published snapshot: `config.Holder`\n\nThe running configuration is a published snapshot held in `config.Holder`\n(`internal/config/holder.go`). Get returns a value under RLock; Set replaces the\nwhole snapshot. A published Config is immutable: reload constructs a fresh one\nand calls Set. `config.Clone` deep-copies every reference-type field so a failed\ndecode cannot mutate the published snapshot's shared maps. `Holder.Get` panics\non a nil receiver — a forgotten `NewHolder` must fail loudly, not boot a daemon\nthat silently runs on an empty config.\n\n### The reloadable-field criterion\n\nThe diff-and-warn on reload asks whether a changed field takes effect without a\nrestart. The criterion is: **a field is reloadable when every consumer re-reads\nit.** That question read as \"what does the daemon re-read\" while the daemon owned\nthe only Holder; it stopped being sufficient when webui began sharing that\nHolder, because webui handlers re-read on every request. The allowlist lives in\n`cmd/archied/reload.go` and is deliberately a deny-by-default list: a field added\nto Config later defaults to requires-restart, forcing whoever adds it to decide\ndeliberately. A reload that changes any non-allowlisted field logs\n\"requires a restart\" with the field names; a fully-applied reload logs Info.\n\n### SIGHUP reload\n\nSIGHUP re-resolves the file config, re-applies the database layer over it\n(`boot.runtimeConfig`, the same call boot makes), validates, and atomically\nrepublishes through the shared Holder. A failed reload keeps the running config\nand records `last_error`/`last_error_at` in `/api/config` so the stale state is\nvisible where the operator is looking. The three poll tickers re-read\n`PollInterval` per iteration and reset on change, so a reloaded interval\ngenuinely takes effect.\n\n### Settings writes go through the control plane\n\nDatabase-backed settings are read and written through\n`ControlPlaneService` on the State Store's gRPC server: catalogue, query,\ncommand, watch. Each feature owns its resource's validation and commands;\nstorage writes versions and audit records only. `docs/prds/runtime-control-plane.md`\nis the specification.\n\nTwo properties matter to anyone changing this area:\n\n- **Bootstrap stays file-owned.** The daemon reads the file config first and\n layers database resources over it at boot (`boot.loadRuntimeConfig`). Values\n the daemon needs before it can reach the State Store — the database path,\n listen addresses, credentials — stay file-only by design.\n- **Reload re-reads them.** SIGHUP resolves the file document and then runs the\n same `boot.runtimeConfig` layering boot does, so a reload cannot revert a\n database-owned setting to its file value. A control plane that cannot answer\n fails the reload rather than publishing the file's values over the running\n ones. Workflow execution budgets arrive on a watch rather than in a resource\n query, so boot records the last published settings and the reload re-applies\n them from there.\n- **Live kinds arrive on a watch.** Provider settings, model role assignments,\n repository policies, scheduling policy, plugin settings, review settings,\n channel settings and container runtime policies are `ApplyMode: live`: the\n daemon keeps a watch per kind (`boot.startRuntimeResourceWatches`,\n `runtimeResourceKinds`) and, on a new version, re-runs `boot.runtimeConfig`\n over the published snapshot and\n republishes through `config.Holder`. A candidate the layered document cannot\n run is refused before it is published — last-known-good keeps running and\n the refusal is reported through apply status. Container runtime policies\n join the live set because their consumers re-read the published config: the\n container pool reads image, pull policy, network, max uptime and concurrency\n cap on every acquire and the daemon resizes its running task dispatcher from\n the same publish; the fields the document cannot carry, `registry_auth`\n above all, stay boot-resolved and remain requires-restart. A kind whose\n consumer lives in another process reconciles there too: `archie-messaging`\n re-reads the stored channel settings on the apply-status restamp interval\n and restarts only the channel whose transport settings changed, while the\n listen addresses it has bound and the set of channels it composed stay\n process bindings that take effect on restart. A kind joins the live list\n only with a consumer that re-reads it; one whose startup-built consumer\n never does stays restart-required, and the model/provider half of the live\n set also rebuilds the gateway chat runtime's provider set and the chat model\n list the chat surfaces offer.\n\n## Completion criteria\n\nThe configuration rearchitecture is complete only when:\n\n- `internal/config` no longer exists;\n- every behavioural value has one classification and owner;\n- every current field and hard-coded value in the ledgers has a recorded\n disposition;\n- domains and plugins receive only their typed settings;\n- all changes use semantic commands and the universal change protocol;\n- active, candidate, rejected, rolled-back, and last-known-good versions are\n queryable with secret-safe provenance;\n- generated documentation lists every external key, default, constraint,\n mutability class, owner, and deprecation;\n- architecture tests prevent a new central runtime configuration model, direct\n live-setting mutation, and incorrectly fixed behavioural values.\n\n## Startup policy: missing credential degrades, invalid config is fatal\n\nDecided 2026-07-30. Recovered 2026-08-09 from the pre-migration issue tracker.\n\nA missing **credential** disables a capability. Only an **invalid config** stops\nthe daemon. \"Forge is a feature, not a requirement.\"\n\nBefore `518a1a0`, an unresolvable forge token did `return 1` in\n`cmd/archied/main.go`, so a chat-only user who never configured GitHub got no\nchat, no gateway, nothing — and under a systemd unit with `Restart=on-failure`\nand no start limit it crash-looped every five seconds with the error scrolling\npast unread. Forge construction now lives in `resolveForge`, shared by the\nprimary forge and every configured identity, and degrades to `forge.NewNoop`\nwith a warning. The identity path had the same defect and was worse: one\nidentity's missing credential killed all the others.\n\nMalformed configuration still fails fast in `configuration.Validate`, well\nbefore this point. That distinction is the whole design.\n\nApply the same reasoning to any future capability credential, LLM keys\nespecially: start, report the capability as unavailable, let the operator see it.\nGuarded by `TestResolveForgeDegradesInsteadOfFailing`.\n\n**Known consequence, tracked separately:** `worktree.Manager` still receives the\nempty token, so pushes fail late rather than at startup.\n\n## Config generation belongs to `archied setup`, not `install.sh`\n\nDecided 2026-07-30 ([#368](https://github.com/samcharles93/archie-core/issues/368)).\nRecovered 2026-08-09 from the pre-migration issue tracker.\n\n`install.sh` must stop generating `config.toml`; `archied setup` owns it. The\ninstaller hand-rolling TOML has already drifted from the schema once: the\ngenerated-config branch hardcoded `key = ARCHIE_GITHUB_TOKEN` and\n`host = ${GITEA_URL}` regardless of the answers given, so choosing Gitea produced\na config naming a token the installer never wrote (archied then exited 1), and\nchoosing GitHub pointed `host` at `gitea.example.com`. `61a5b35` patched this\nwith a `forge_block` shell helper shared by both config paths — that helper is a\n**workaround**, deleted by\n[#72](https://github.com/samcharles93/archie-core/issues/72). The durable fix is\nthat the code writing the config is the code that reads it.\n\nWhy Go and not Python: the installer must run where nothing is installed yet, and\n`python3` with `tomllib` means 3.11+, which is not safe on LTS distros. `archied`\nis already being built two steps earlier.\n\n**Key constraint for the port:** `BurntSushi/toml` does **not** survive a\ndecode/encode round trip with comments intact, and `config.example.toml` carries\ndocumentation users are meant to hand-edit. Decide the write strategy\n([#412](https://github.com/samcharles93/archie-core/issues/412)) before writing\nany prompting code. Do not port a separate `config.yaml`/`auth.yaml` secret\nsplit — Archie already has `secret.SecretRef{engine,key}`, so setup writes the\nreference to TOML and the value to the secret engine.\n" }, { "url": "/dev/architecture/dashboard-agent-tools/", diff --git a/internal/app/archied/bootstrap.go b/internal/app/archied/bootstrap.go index 574dc4280..fde701270 100644 --- a/internal/app/archied/bootstrap.go +++ b/internal/app/archied/bootstrap.go @@ -860,7 +860,7 @@ func (b *boot) startEmbeddedNATS(ctx context.Context) (string, string, error) { // not require Docker; repository tasks park instead of falling back to a host // model loop. func (b *boot) setupContainers(ctx context.Context) func() { - containerPool, storeBackend, closeDocker := startContainers(ctx, b.cfg, b.secrets, b.log) + containerPool, storeBackend, closeDocker := startContainers(ctx, b.cfgHolder, b.secrets, b.log) b.containerPool = containerPool b.storeBackend = storeBackend return closeDocker diff --git a/internal/app/archied/bootstrap_nats_test.go b/internal/app/archied/bootstrap_nats_test.go index ab535fcba..aa5b107f9 100644 --- a/internal/app/archied/bootstrap_nats_test.go +++ b/internal/app/archied/bootstrap_nats_test.go @@ -109,7 +109,7 @@ func TestConnectEmbeddedNATSUsesManagedWorkerBridge(t *testing.T) { DockerClient: dockerClient, Network: "workers", RequireHostGateway: true, - }, slog.New(slog.DiscardHandler)) + }, nil, slog.New(slog.DiscardHandler)) if err != nil { t.Fatalf("container.NewPool = %v", err) } diff --git a/internal/app/archied/control_plane.go b/internal/app/archied/control_plane.go index 6fc969e7e..ac235f54b 100644 --- a/internal/app/archied/control_plane.go +++ b/internal/app/archied/control_plane.go @@ -92,6 +92,10 @@ func (b *boot) reloadConfig(ctx context.Context, doc *configuration.Document) er old := b.cfgHolder.Get() b.currentProvenance.Store(&doc.Provenance) b.publishConfig(ctx, cfg) + // The container pool reads the published config on every acquire, but the + // dispatcher holds its limit; a reloaded containers.max_concurrency must + // reach it the same way a control-plane update does. + b.resizeTaskDispatcher(cfg.Containers.MaxConcurrency) if fields := changedNonReloadableFields(old, cfg); len(fields) > 0 { b.log.Warn("config reloaded; some changes require a restart", "fields", fields, "paths", doc.Provenance.Paths()) @@ -260,10 +264,14 @@ func (b *boot) startLiveSettings(ctx context.Context) error { // (b.runtimeConfig, which applies the execution budgets the settings watch // published) and republishes through config.Holder, so the change takes // effect without a restart. Kinds stay out of this list while a -// startup-built component still holds their value -- tool and container +// startup-built component still holds their value -- tool // settings are frozen in components built at boot and remain // restart-required -- and a kind joins it only with a consumer that re-reads -// it. PluginSettingsKind is on the list because its only consumer is the +// it. ContainerRuntimePoliciesKind is on the list because both consumers +// re-read the published config: the container pool takes its image, pull +// policy, network, max-uptime and concurrency cap per acquire, and the +// daemon's dispatcher is resized after the publish (applyRuntimeResourceUpdate +// below). PluginSettingsKind is on the list because its only consumer is the // directory reconciliation, which reads the running config each tick; a // removed plugin or engine still cannot unload, so that kind reports the // removal rather than requiring a restart. ChannelSettingsKind is on the list @@ -279,6 +287,7 @@ var runtimeResourceKinds = []string{ controlplane.PluginSettingsKind, controlplane.ReviewSettingsKind, controlplane.ChannelSettingsKind, + controlplane.ContainerRuntimePoliciesKind, } // startRuntimeResourceWatches keeps a watch per live kind established for the @@ -362,6 +371,22 @@ func (b *boot) applyRuntimeResourceUpdate(ctx context.Context, kind string, upda // report the reconciliation's own outcome, which is the one that knows // whether a removal is outstanding. b.reconcileRuntimePlugins(ctx) + case controlplane.ContainerRuntimePoliciesKind: + // The pool reads the republished config itself; the dispatcher holds + // its limit, so a raise must be woken to admit queued work now and a + // lower must be honoured as running tasks finish. + b.resizeTaskDispatcher(cfg.Containers.MaxConcurrency) + } +} + +// resizeTaskDispatcher applies a new containers.max_concurrency to the +// daemon's running dispatcher. The pool needs no poke because it reads the +// published config on every acquire. b.d is nil in the boot window in which +// the watches are already live but the daemon is not built yet; there the +// dispatcher is created later from the config the next drain reads. +func (b *boot) resizeTaskDispatcher(maxConcurrency int) { + if b.d != nil { + b.d.ResizeTaskDispatcher(maxConcurrency) } } diff --git a/internal/app/archied/control_plane_resources_test.go b/internal/app/archied/control_plane_resources_test.go index 7295d03d6..0609d34a6 100644 --- a/internal/app/archied/control_plane_resources_test.go +++ b/internal/app/archied/control_plane_resources_test.go @@ -15,6 +15,7 @@ import ( "github.com/samcharles93/archie-core/internal/agentexec" "github.com/samcharles93/archie-core/internal/app/controlplane" pb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1" + "github.com/samcharles93/archie-core/internal/daemon" "github.com/samcharles93/archie-core/internal/secret" ) @@ -240,3 +241,33 @@ func TestLiveModelRoleUpdateRebuildsTheChatModelRuntime(t *testing.T) { }) } } + +// TestLiveContainerRuntimePoliciesReLayerAndResizeTheDispatcher pins the +// container kind's live seam (archie-core-zfb0.2) at the apply path: a stored +// container-runtime-policies document re-layers into the running config the +// pool reads per acquire, and the daemon resizes its running dispatcher from +// the same publish rather than requiring a restart. +func TestLiveContainerRuntimePoliciesReLayerAndResizeTheDispatcher(t *testing.T) { + values := databaseOwnedResources() + values[controlplane.ContainerRuntimePoliciesKind] = map[string]any{ + "image": "archie:live", "max_concurrency": 5, "pull_policy": "missing", + } + stub := &resourceWatchStub{values: values, version: 3, storedVersion: map[string]int64{}, opens: map[string]int{}} + b, _ := newResourceWatchBoot(t, stub) + if err := b.loadRuntimeConfig(t.Context()); err != nil { + t.Fatalf("loadRuntimeConfig: %v", err) + } + // A running daemon, so the resize arm executes against a real dispatcher + // rather than taking the boot-window nil guard. + b.d = &daemon.Daemon{Cfg: b.cfgHolder, Log: b.log} + + b.applyRuntimeResourceUpdate(t.Context(), controlplane.ContainerRuntimePoliciesKind, controlplane.AppliedResource{Version: 4}) + + got := b.cfgHolder.Get().Containers + if got.MaxConcurrency != 5 { + t.Errorf("running max_concurrency = %d, want the stored 5", got.MaxConcurrency) + } + if got.Image != "archie:live" { + t.Errorf("running image = %q, want the stored archie:live", got.Image) + } +} diff --git a/internal/app/archied/main.go b/internal/app/archied/main.go index ae61a6aed..a61f46b25 100644 --- a/internal/app/archied/main.go +++ b/internal/app/archied/main.go @@ -974,11 +974,12 @@ func safePluginInfo(p plugin.Plugin) (name, version string) { // there is deliberately no host execution fallback. func startContainers( ctx context.Context, - cfg config.Config, + cfgHolder *config.Holder, secrets *secret.Registry, log *slog.Logger, ) (*container.Pool, storage.Backend, func()) { noop := func() {} + cfg := cfgHolder.Get() // Single Docker client shared between pool and storage backend. dockerCli, err := client.New(client.FromEnv) if err != nil { @@ -1002,7 +1003,7 @@ func startContainers( // Only the embedded broker binds a discovered host gateway. The // standalone State Store uses its configured, container-reachable target. RequireHostGateway: cfg.NATS.Mode == config.NATSModeEmbedded, - }, log) + }, containerLiveSettings(cfgHolder), log) if err != nil { // A missing image is recoverable by hand. log.Error("autonomous workflows unavailable: container pool unavailable", "err", err, @@ -1021,6 +1022,26 @@ func startContainers( } } +// containerLiveSettings reads the per-acquire container settings from the +// running config. The pool calls it on every acquire, so a stored +// container-runtime-policies change reaches the next container without a +// restart. Only fields the resource document carries are taken live: +// RegistryAuth is file-owned and stays the boot-resolved value, DockerClient +// and RequireHostGateway are construction-time, and GracePeriod is not part +// of the document. +func containerLiveSettings(cfgHolder *config.Holder) func() container.Config { + return func() container.Config { + cfg := cfgHolder.Get() + return container.Config{ + Image: cfg.Containers.Image, + MaxConcurrency: cfg.Containers.MaxConcurrency, + MaxUptime: cfg.Containers.MaxUptime.Std(), + PullPolicy: cfg.Containers.PullPolicy, + Network: cfg.Containers.Network, + } + } +} + // resolveRegistryAuth resolves [containers].registry_auth into the credential // the container pool sends on pull. It returns "" -- an anonymous pull, the // pool's behaviour before the field existed -- whenever the credential is not diff --git a/internal/app/archied/reload.go b/internal/app/archied/reload.go index edaa23f82..6f43202f0 100644 --- a/internal/app/archied/reload.go +++ b/internal/app/archied/reload.go @@ -156,17 +156,21 @@ var reloadableFields = map[string]bool{ // "Selection": a Kit profile applies without a restart. Credentials is the // same shape one layer down: kitrun.Launcher reads Config.Get().Containers. // Credentials fresh on every Launch (CredentialBindingsKind), never a slice -// captured once when the launcher was built. Everything else in -// Containers (Image, MaxConcurrency, MaxUptime, PullPolicy, Network, -// RegistryAuth) is frozen in the startup-built container pool -- the -// dispatchers re-read MaxConcurrency but the pool captures it at construction -// (container/pool.go:94,163), so a change only partially applies and must -// warn requires-restart. Forge.Host is carried into TaskContext by ForTask +// captured once when the launcher was built. Image, MaxConcurrency, MaxUptime, +// PullPolicy and Network are re-read per acquire by the container pool and +// MaxConcurrency also resizes the running dispatcher on publish +// (archie-core-zfb0.2), so a reload applies them without a restart. +// RegistryAuth stays frozen in the startup-built pool: it is file-owned and +// never part of the container-runtime-policies document, so a changed value +// must warn requires-restart. Forge.Host is carried into TaskContext by ForTask // (display/link building only); the forge client itself is startup-built, so // Type/Token/TokenEnv stay requires-restart. var reloadableSubFields = map[string]map[string]bool{ - "Containers": {"VolumeTTL": true, "Profiles": true, "Credentials": true}, - "Forge": {"Host": true}, + "Containers": { + "VolumeTTL": true, "Profiles": true, "Credentials": true, + "Image": true, "MaxConcurrency": true, "MaxUptime": true, "PullPolicy": true, "Network": true, + }, + "Forge": {"Host": true}, // Policy is carried into TaskConfig by ForTask (config.go); MCPServers, // WebFetch and Minimax are not and stay requires-restart. "Tools": {"Policy": true}, diff --git a/internal/app/archied/reload_test.go b/internal/app/archied/reload_test.go index a10241b80..89224f7f8 100644 --- a/internal/app/archied/reload_test.go +++ b/internal/app/archied/reload_test.go @@ -198,28 +198,31 @@ func TestChangedNonReloadableFields(t *testing.T) { t.Fatalf("reloadable-only: got %v, want []", got) } - // Containers sub-field granularity: Image change warns, MaxConcurrency - // and VolumeTTL changes do not. + // Containers sub-field granularity: the fields the pool re-reads per + // acquire, and the concurrency cap the dispatcher resizes on, are + // reloadable; RegistryAuth stays frozen in the startup-built pool. c1 := base c1.Containers = config.ContainerConfig{Image: "img1", MaxConcurrency: 2, VolumeTTL: config.Duration(24 * time.Hour)} c2 := c1 c2.Containers.Image = "img2" - if got := changedNonReloadableFields(c1, c2); len(got) != 1 || got[0] != "Containers.Image" { - t.Fatalf("Image change: got %v, want [Containers.Image]", got) + if got := changedNonReloadableFields(c1, c2); len(got) != 0 { + t.Fatalf("Image change: got %v, want [] (the pool reads it per acquire)", got) } c3 := c1 c3.Containers.MaxConcurrency = 4 - // MaxConcurrency is NOT reloadable: the dispatchers re-read it but the - // startup-built container pool captures it (container/pool.go:94), so - // a change only partially applies and must warn requires-restart. - if got := changedNonReloadableFields(c1, c3); len(got) != 1 || got[0] != "Containers.MaxConcurrency" { - t.Fatalf("MaxConcurrency change: got %v, want [Containers.MaxConcurrency]", got) + if got := changedNonReloadableFields(c1, c3); len(got) != 0 { + t.Fatalf("MaxConcurrency change: got %v, want [] (the pool reads it and the dispatcher resizes)", got) } c4 := c1 c4.Containers.VolumeTTL = config.Duration(48 * time.Hour) if got := changedNonReloadableFields(c1, c4); len(got) != 0 { t.Fatalf("VolumeTTL change: got %v, want []", got) } + c5 := c1 + c5.Containers.RegistryAuth = config.SecretRef{Engine: "env", Key: "REGISTRY_AUTH"} + if got := changedNonReloadableFields(c1, c5); len(got) != 1 || got[0] != "Containers.RegistryAuth" { + t.Fatalf("RegistryAuth change: got %v, want [Containers.RegistryAuth] (the pool resolves it at boot)", got) + } // NATS is not reloadable: a URL change warns (the daemon's client is // startup-built; containers get the frozen ConnectedNATS). diff --git a/internal/app/archied/state_store_regression_test.go b/internal/app/archied/state_store_regression_test.go index fd6901745..8661f028d 100644 --- a/internal/app/archied/state_store_regression_test.go +++ b/internal/app/archied/state_store_regression_test.go @@ -47,7 +47,7 @@ func TestExternalServicesDoNotRequireBridgeNetwork(t *testing.T) { cfg.NATS.Mode = config.NATSModeExternal cfg.Containers.Network = "host" cfg.Services = config.Services{config.ServiceNameState: {Target: "state.example.test:9090"}} - pool, _, cleanup := startContainers(t.Context(), cfg, secret.NewRegistry(), slog.New(slog.DiscardHandler)) + pool, _, cleanup := startContainers(t.Context(), config.NewHolder(cfg), secret.NewRegistry(), slog.New(slog.DiscardHandler)) defer cleanup() if pool == nil { t.Fatal("external services on host networking must construct a worker pool") diff --git a/internal/app/controlplane/controlplane_test.go b/internal/app/controlplane/controlplane_test.go index 51484718b..72c750c34 100644 --- a/internal/app/controlplane/controlplane_test.go +++ b/internal/app/controlplane/controlplane_test.go @@ -197,11 +197,11 @@ func TestRegistryRoutesAndValidatesDefinitions(t *testing.T) { found := false for _, descriptor := range catalog.Resources { if descriptor.Kind == ContainerRuntimePoliciesKind { - found = descriptor.ApplyMode == "restart-required" + found = descriptor.ApplyMode == "live" } } if !found { - t.Fatal("container runtime policies missing restart-required descriptor") + t.Fatal("container runtime policies missing live descriptor") } _, err = server.Command(t.Context(), &pb.CommandRequest{Kind: ContainerRuntimePoliciesKind, Command: "replace", ValueJson: []byte(`{"pull_policy":"sometimes"}`), Actor: "test", Source: "test", RequestId: "bad", ExpectedVersion: 1}) diff --git a/internal/app/controlplane/operational_settings.go b/internal/app/controlplane/operational_settings.go index 533757e8f..3f0af7f3d 100644 --- a/internal/app/controlplane/operational_settings.go +++ b/internal/app/controlplane/operational_settings.go @@ -54,7 +54,13 @@ func operationalDefinitions() []Definition { {Kind: PluginSettingsKind, Title: "Plugin settings", ApplyMode: "live", Document: pluginSettings{}, Seed: func(cfg config.Config) any { return pluginSettings{cfg.PluginDir, cfg.ModuleDir, cfg.SecretEngineDir, cfg.SkillsDir} }, Validate: validatePluginSettings}, - {Kind: ContainerRuntimePoliciesKind, Title: "Container runtime policies", ApplyMode: "restart-required", Document: containerRuntimePolicies{}, Seed: seedContainerPolicies, Validate: validateContainers, Normalize: normalizeContainerPolicies}, + // ApplyMode is live: the container pool reads the published config on + // every acquire (image, pull policy, network, max uptime, concurrency + // cap) and the daemon resizes its running dispatcher after the publish, + // so neither a deferred container nor a queued task needs a restart + // (archie-core-zfb0.2). RegistryAuth is file-owned and never part of + // this document, so it stays a boot-resolved value. + {Kind: ContainerRuntimePoliciesKind, Title: "Container runtime policies", ApplyMode: "live", Document: containerRuntimePolicies{}, Seed: seedContainerPolicies, Validate: validateContainers, Normalize: normalizeContainerPolicies}, } } diff --git a/internal/app/controlplane/runtime_config_test.go b/internal/app/controlplane/runtime_config_test.go index f1757a2f2..22b4b9b73 100644 --- a/internal/app/controlplane/runtime_config_test.go +++ b/internal/app/controlplane/runtime_config_test.go @@ -553,13 +553,13 @@ func TestRuntimeResourceKindsApplyLive(t *testing.T) { modes[definition.Kind] = definition.ApplyMode } for _, kind := range []string{ - ProviderSettingsKind, ModelRoleAssignmentsKind, RepositoryPoliciesKind, SchedulingPolicyKind, AgentProfileKind, CredentialBindingsKind, PluginSettingsKind, ReviewSettingsKind, ChannelSettingsKind, + ProviderSettingsKind, ModelRoleAssignmentsKind, RepositoryPoliciesKind, SchedulingPolicyKind, AgentProfileKind, CredentialBindingsKind, PluginSettingsKind, ReviewSettingsKind, ChannelSettingsKind, ContainerRuntimePoliciesKind, } { if modes[kind] != "live" { t.Errorf("%s applies %q, want live: the daemon re-layers this kind on a watch", kind, modes[kind]) } } - for _, kind := range []string{ToolSettingsKind, ContainerRuntimePoliciesKind} { + for _, kind := range []string{ToolSettingsKind} { if modes[kind] != "restart-required" { t.Errorf("%s applies %q, want restart-required: a startup-built component still holds it", kind, modes[kind]) } diff --git a/internal/container/network_test.go b/internal/container/network_test.go index 247797158..fcaea0640 100644 --- a/internal/container/network_test.go +++ b/internal/container/network_test.go @@ -56,7 +56,7 @@ func TestNewPoolResolvesDefaultBridgeGatewayForNativeDaemon(t *testing.T) { pool, err := NewPool(t.Context(), Config{ DockerClient: dockerClient, RequireHostGateway: true, - }, discardLogger()) + }, nil, discardLogger()) if err != nil { t.Fatalf("NewPool = %v", err) } @@ -138,7 +138,7 @@ func TestNewPoolRejectsUnreachableEmbeddedNetwork(t *testing.T) { DockerClient: dockerClient, Network: "workers", RequireHostGateway: true, - }, discardLogger()) + }, nil, discardLogger()) if err == nil { _ = pool.Close() t.Fatal("NewPool = nil error, want embedded reachability validation failure") diff --git a/internal/container/pool.go b/internal/container/pool.go index f450b687c..6e3650864 100644 --- a/internal/container/pool.go +++ b/internal/container/pool.go @@ -98,6 +98,12 @@ type Pool struct { // the listener to an address visible only to workers on this bridge. hostGateway string + // live, when set, returns the pool's current settings. A production pool + // is given one that reads the running config, so a container-runtime- + // policies change reaches the next acquire without a restart; a test pool + // leaves it nil and cfg is the whole truth. + live func() Config + mu sync.Mutex active int // pulled records the images already made available, so a profile's image @@ -108,6 +114,25 @@ type Pool struct { teardowns map[string]*containerTeardown } +// current returns the settings a new acquire should use: the live source when +// the pool has one, otherwise the construction-time Config. Only the fields a +// container-runtime-policies document can carry are taken from the live +// source; DockerClient, RequireHostGateway, RegistryAuth and GracePeriod stay +// construction-time, because none of them is part of that document. +func (p *Pool) current() Config { + if p.live == nil { + return p.cfg + } + cfg := p.cfg + live := p.live() + cfg.Image = live.Image + cfg.MaxConcurrency = live.MaxConcurrency + cfg.MaxUptime = live.MaxUptime + cfg.PullPolicy = live.PullPolicy + cfg.Network = live.Network + return cfg +} + // Config is the subset of daemon container configuration the pool needs. type Config struct { Image string @@ -142,7 +167,14 @@ type Config struct { // // If cfg.DockerClient is set, it is reused (caller owns Close). // Otherwise a new client is created via client.FromEnv (pool owns Close). -func NewPool(ctx context.Context, cfg Config, log *slog.Logger) (*Pool, error) { +// +// live, when non-nil, is the pool's live source of the per-acquire settings a +// container-runtime-policies document can change (Image, MaxConcurrency, +// MaxUptime, PullPolicy, Network). It is read on every acquire, so a stored +// change reaches the next container without a restart; nil means cfg is fixed +// for the pool's life. DockerClient, RequireHostGateway, RegistryAuth and +// GracePeriod stay construction-time because the document never carries them. +func NewPool(ctx context.Context, cfg Config, live func() Config, log *slog.Logger) (*Pool, error) { cli := cfg.DockerClient if cli == nil { var err error @@ -173,13 +205,16 @@ func NewPool(ctx context.Context, cfg Config, log *slog.Logger) (*Pool, error) { p := &Pool{ cli: cli, cfg: cfg, + live: live, log: log, ownCli: cfg.DockerClient == nil, network: network, hostGateway: hostGateway, } - // Pull image if needed. + // Pull image if needed. The configured image is recorded as pulled, so an + // acquire that names it skips a second pull even though the live settings + // may have changed the configured image since. if cfg.PullPolicy == "always" || cfg.PullPolicy == "missing" { if err := p.pullImage(ctx, cfg.Image); err != nil { if err := cli.Close(); err != nil { @@ -187,6 +222,7 @@ func NewPool(ctx context.Context, cfg Config, log *slog.Logger) (*Pool, error) { } return nil, err } + p.markPulled(cfg.Image) } // Recover orphaned containers from a previous daemon crash. @@ -201,13 +237,17 @@ func NewPool(ctx context.Context, cfg Config, log *slog.Logger) (*Pool, error) { // is set, the pool schedules a hard stop and remove of the container once // that lifetime cap elapses, regardless of task state. func (p *Pool) Acquire(ctx context.Context, image string, mounts []storage.Mount, env []string) (*Container, error) { + // One snapshot for the whole acquire: the image, pull policy, network, + // concurrency cap and max-uptime cap are all read once, so a settings + // change landing mid-acquire cannot mix two versions. + cfg := p.current() if image == "" { - image = p.cfg.Image + image = cfg.Image } - if err := p.EnsureImage(ctx, image); err != nil { + if err := p.ensureImage(ctx, cfg, image); err != nil { return nil, err } - if err := p.reserve(); err != nil { + if err := p.reserve(cfg.MaxConcurrency); err != nil { return nil, err } @@ -217,12 +257,18 @@ func (p *Pool) Acquire(ctx context.Context, image string, mounts []storage.Mount Mounts: storage.ConvertMounts(mounts), AutoRemove: true, } - if p.network != "" { + // The live network wins when it names one; otherwise the network resolved + // at construction (the configured one, or self-detection) stays in force. + network := cfg.Network + if network == "" { + network = p.network + } + if network != "" { // Join the same user-defined network the daemon itself is on, so // the agent container can resolve sibling compose services (nats, // etc.) by name -- otherwise Docker attaches it to the default // bridge network, where those hostnames don't resolve. - hostConfig.NetworkMode = container.NetworkMode(p.network) + hostConfig.NetworkMode = container.NetworkMode(network) } resp, err := p.cli.ContainerCreate(ctx, client.ContainerCreateOptions{ @@ -249,16 +295,17 @@ func (p *Pool) Acquire(ctx context.Context, image string, mounts []storage.Mount p.unreserve() return nil, fmt.Errorf("container start: %w", err) } - return p.started(ctx, resp.ID, name), nil + return p.started(ctx, cfg.MaxUptime, resp.ID, name), nil } // AcquireKit starts a Kit task container under the same concurrency cap, // lifetime cap and exit watch as Acquire. Release tears it down. func (p *Pool) AcquireKit(ctx context.Context, s KitSpec) (*Container, error) { - if err := p.EnsureImage(ctx, s.Image); err != nil { + cfg := p.current() + if err := p.ensureImage(ctx, cfg, s.Image); err != nil { return nil, err } - if err := p.reserve(); err != nil { + if err := p.reserve(cfg.MaxConcurrency); err != nil { return nil, err } s.Labels = maps.Clone(s.Labels) @@ -271,14 +318,14 @@ func (p *Pool) AcquireKit(ctx context.Context, s KitSpec) (*Container, error) { p.unreserve() return nil, err } - return p.started(ctx, id, s.Name), nil + return p.started(ctx, cfg.MaxUptime, id, s.Name), nil } -func (p *Pool) reserve() error { +func (p *Pool) reserve(maxConcurrency int) error { p.mu.Lock() defer p.mu.Unlock() - if p.cfg.MaxConcurrency > 0 && p.active >= p.cfg.MaxConcurrency { - return fmt.Errorf("max concurrency %d reached", p.cfg.MaxConcurrency) + if maxConcurrency > 0 && p.active >= maxConcurrency { + return fmt.Errorf("max concurrency %d reached", maxConcurrency) } p.active++ return nil @@ -290,9 +337,9 @@ func (p *Pool) unreserve() { p.mu.Unlock() } -func (p *Pool) started(ctx context.Context, id, name string) *Container { - if p.cfg.MaxUptime > 0 { - p.armMaxUptime(ctx, id) +func (p *Pool) started(ctx context.Context, maxUptime time.Duration, id, name string) *Container { + if maxUptime > 0 { + p.armMaxUptime(ctx, maxUptime, id) } p.log.Info("container started", "id", id[:12], "name", name) return &Container{ID: id, exited: p.watchExit(ctx, id)} @@ -348,12 +395,12 @@ type containerTeardown struct { // The lock covers only bookkeeping: the callback runs on its own goroutine and // nothing here waits on it, and every Docker call it makes happens after it // has released p.mu. -func (p *Pool) armMaxUptime(ctx context.Context, id string) { +func (p *Pool) armMaxUptime(ctx context.Context, maxUptime time.Duration, id string) { p.mu.Lock() defer p.mu.Unlock() t := p.teardownLocked(id) - t.timer = time.AfterFunc(p.cfg.MaxUptime, func() { + t.timer = time.AfterFunc(maxUptime, func() { p.reapMaxUptime(ctx, id) }) } @@ -544,11 +591,18 @@ func (p *Pool) Close() error { // ── helpers ────────────────────────────────────────────────────────── -// EnsureImage pulls an image other than the configured one the first time a -// task asks for it, under the configured pull policy. The configured image -// was already pulled by NewPool. +// EnsureImage pulls ref under the live pull policy unless the pool has +// already made it available. The configured image was pulled by NewPool and +// recorded, so a caller naming it skips a second pull. func (p *Pool) EnsureImage(ctx context.Context, ref string) error { - if ref == p.cfg.Image || (p.cfg.PullPolicy != "always" && p.cfg.PullPolicy != "missing") { + return p.ensureImage(ctx, p.current(), ref) +} + +// ensureImage pulls ref under cfg's pull policy unless it is already present +// in the pool. It is the live read an acquire uses; EnsureImage is the +// exported form for callers that only name a ref. +func (p *Pool) ensureImage(ctx context.Context, cfg Config, ref string) error { + if cfg.PullPolicy != "always" && cfg.PullPolicy != "missing" { return nil } p.mu.Lock() @@ -560,19 +614,26 @@ func (p *Pool) EnsureImage(ctx context.Context, ref string) error { if err := p.pullImage(ctx, ref); err != nil { return err } + p.markPulled(ref) + return nil +} + +// markPulled records that ref is available locally, so a later acquire skips +// a second pull. +func (p *Pool) markPulled(ref string) { p.mu.Lock() if p.pulled == nil { p.pulled = map[string]bool{} } p.pulled[ref] = true p.mu.Unlock() - return nil } -// pullImage pulls ref. On "missing" policy, skips if the image already -// exists locally. +// pullImage pulls ref under the live pull policy. On "missing" policy, skips +// if the image already exists locally. func (p *Pool) pullImage(ctx context.Context, ref string) error { - if p.cfg.PullPolicy == "missing" { + cfg := p.current() + if cfg.PullPolicy == "missing" { _, err := p.cli.ImageInspect(ctx, ref) if err == nil { p.log.Info("image already present", "image", ref) @@ -581,7 +642,7 @@ func (p *Pool) pullImage(ctx context.Context, ref string) error { } p.log.Info("pulling image", "image", ref) - rc, err := p.cli.ImagePull(ctx, ref, client.ImagePullOptions{RegistryAuth: p.registryAuthHeader(ref)}) + rc, err := p.cli.ImagePull(ctx, ref, client.ImagePullOptions{RegistryAuth: p.registryAuthHeader(cfg, ref)}) if err != nil { return fmt.Errorf("pull %s: %w", ref, err) } @@ -606,11 +667,11 @@ func (p *Pool) pullImage(ctx context.Context, ref string) error { // autonomous task down with it, and the warning is what makes the 401 a private // registry will still answer explicable. A missing credential is silent -- it is // the deployment shape that has no private registry. -func (p *Pool) registryAuthHeader(ref string) string { - if p.cfg.RegistryAuth == "" { +func (p *Pool) registryAuthHeader(cfg Config, ref string) string { + if cfg.RegistryAuth == "" { return "" } - encoded, err := encodeRegistryAuth(p.cfg.RegistryAuth) + encoded, err := encodeRegistryAuth(cfg.RegistryAuth) if err != nil { p.log.Warn("registry credential unusable; pulling without auth", "image", ref, "err", err) return "" @@ -700,8 +761,8 @@ func (p *Pool) Active() int { return p.active } -// Cap is the concurrency cap this pool enforces, from the configured +// Cap is the concurrency cap this pool enforces, from the live // containers.max_concurrency. Zero means unlimited (Acquire only enforces a // cap when MaxConcurrency > 0), so a caller rendering "active/cap" must not // treat zero as a cap of zero. -func (p *Pool) Cap() int { return p.cfg.MaxConcurrency } +func (p *Pool) Cap() int { return p.current().MaxConcurrency } diff --git a/internal/container/pool_test.go b/internal/container/pool_test.go index f6d95971c..c784325c2 100644 --- a/internal/container/pool_test.go +++ b/internal/container/pool_test.go @@ -12,6 +12,7 @@ import ( "os" "path/filepath" "strings" + "sync" "sync/atomic" "testing" "time" @@ -844,3 +845,158 @@ func TestPullImageWithoutUsableRegistryCredentialPullsAnonymously(t *testing.T) }) } } + +// ── live settings (archie-core-zfb0.2) ──────────────────────────────── + +// TestPoolReadsSettingsPerAcquire pins that a container-runtime-policies +// change reaches the next acquire without a restart. The pool's image, pull +// policy, network, concurrency cap and max-uptime cap must come from the live +// settings source rather than the value captured when the pool was built; a +// construction-time network stays the fallback when the live value is empty. +func TestPoolReadsSettingsPerAcquire(t *testing.T) { + type createBody struct { + Image string `json:"Image"` + HostConfig struct { + NetworkMode string `json:"NetworkMode"` + } `json:"HostConfig"` + } + + var ( + mu sync.Mutex + creates []createBody + pulls []string + ) + var stopped, removed atomic.Int32 + + dockerAPI := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/containers/create"): + var body createBody + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + mu.Lock() + creates = append(creates, body) + id := fmt.Sprintf("%064d", len(creates)) + mu.Unlock() + writeDockerJSON(t, w, map[string]any{"Id": id, "Warnings": []string{}}) + case strings.HasSuffix(r.URL.Path, "/start"): + w.WriteHeader(http.StatusNoContent) + case strings.HasSuffix(r.URL.Path, "/stop"): + stopped.Add(1) + w.WriteHeader(http.StatusNoContent) + case strings.HasSuffix(r.URL.Path, "/images/create"): + mu.Lock() + pulls = append(pulls, r.URL.Query().Get("fromImage")) + mu.Unlock() + writeDockerJSON(t, w, map[string]any{}) + case strings.HasSuffix(r.URL.Path, "/json"): + http.Error(w, "no such image", http.StatusNotFound) + case r.Method == http.MethodDelete: + removed.Add(1) + w.WriteHeader(http.StatusNoContent) + default: + http.Error(w, "unexpected Docker API path "+r.URL.Path, http.StatusNotFound) + } + })) + t.Cleanup(dockerAPI.Close) + + dockerClient, err := client.New(client.WithHost(dockerAPI.URL), client.WithAPIVersion("1.55")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = dockerClient.Close() }) + + var settingsMu sync.Mutex + current := Config{ + Image: "old/image", PullPolicy: "missing", Network: "", + MaxConcurrency: 1, MaxUptime: 10 * time.Second, + } + setCurrent := func(c Config) { + settingsMu.Lock() + current = c + settingsMu.Unlock() + } + + pool := &Pool{ + cli: dockerClient, + cfg: current, + network: "detected", + live: func() Config { + settingsMu.Lock() + defer settingsMu.Unlock() + return current + }, + // NewPool pulls the configured image before the first acquire. + pulled: map[string]bool{"old/image": true}, + log: discardLogger(), + } + + acquire := func(t *testing.T) { + t.Helper() + if _, err := pool.Acquire(context.Background(), "", nil, nil); err != nil { + t.Fatalf("Acquire: %v", err) + } + } + created := func(t *testing.T, i int) createBody { + t.Helper() + mu.Lock() + defer mu.Unlock() + if i >= len(creates) { + t.Fatalf("only %d create calls, want at least %d", len(creates), i+1) + } + return creates[i] + } + + acquire(t) + if got := created(t, 0); got.Image != "old/image" || got.HostConfig.NetworkMode != "detected" { + t.Fatalf("first acquire used image %q network %q, want the construction settings", got.Image, got.HostConfig.NetworkMode) + } + + setCurrent(Config{ + Image: "new/image", PullPolicy: "missing", Network: "newnet", + MaxConcurrency: 2, MaxUptime: 60 * time.Millisecond, + }) + acquire(t) + second := created(t, 1) + if second.Image != "new/image" { + t.Errorf("second acquire used image %q, want the live image", second.Image) + } + if second.HostConfig.NetworkMode != "newnet" { + t.Errorf("second acquire joined network %q, want the live network", second.HostConfig.NetworkMode) + } + if got := pool.Cap(); got != 2 { + t.Errorf("Cap() = %d, want the live MaxConcurrency 2", got) + } + mu.Lock() + pulledNew := false + for _, ref := range pulls { + if strings.HasSuffix(ref, "new/image") { + pulledNew = true + } + } + mu.Unlock() + if !pulledNew { + t.Errorf("pulls = %v, want the live image pulled", pulls) + } + + // The live max-uptime cap, not the construction-time 10s, reaps it. + deadline := time.Now().Add(2 * time.Second) + for (stopped.Load() == 0 || removed.Load() == 0) && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + if stopped.Load() == 0 || removed.Load() == 0 { + t.Errorf("live max uptime did not reap the second container: stopped=%d removed=%d", stopped.Load(), removed.Load()) + } + + // An empty live network keeps the construction-time fallback in force. + setCurrent(Config{ + Image: "new/image", PullPolicy: "missing", Network: "", + MaxConcurrency: 3, MaxUptime: 10 * time.Second, + }) + acquire(t) + if got := created(t, 2); got.HostConfig.NetworkMode != "detected" { + t.Errorf("third acquire joined network %q, want the construction-time fallback %q", got.HostConfig.NetworkMode, "detected") + } +} diff --git a/internal/daemon/container_env_test.go b/internal/daemon/container_env_test.go index 16fa658c8..775376ccd 100644 --- a/internal/daemon/container_env_test.go +++ b/internal/daemon/container_env_test.go @@ -68,7 +68,7 @@ func TestConnectedNATSEndpointReachesManagedContainerCreate(t *testing.T) { Network: networkName, DockerClient: dockerClient, RequireHostGateway: true, - }, slog.New(slog.DiscardHandler)) + }, nil, slog.New(slog.DiscardHandler)) if err != nil { t.Fatalf("container.NewPool = %v", err) } diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index 0a4e88774..54cdd1b7c 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -199,6 +199,12 @@ type Daemon struct { // execution is unavailable and process parks tasks; every runnable task gets // a fresh container. ContainerPool *container.Pool + // dispatcher bounds task execution globally across poll passes. Built once, + // on first use, so ResizeTaskDispatcher can apply a live + // containers.max_concurrency change without a restart; drainNATS submits + // each pass through it and waits for that pass. + dispatcherOnce sync.Once + dispatcher *taskDispatcher // Storage is the pluggable storage backend for container mounts. // A runnable task requires it; acquireTaskContainer parks on nil. Storage storage.Backend @@ -975,7 +981,7 @@ func (d *Daemon) botUserForTask(task *workflowtask.Task) string { // requeued tasks (waiting_human approval, retry-parked) that didn't come // through a NATS publish. func (d *Daemon) drainNATS(ctx context.Context) { - dispatcher := newTaskDispatcher(d.Cfg.Get().Containers.MaxConcurrency, d.allowConcurrentForTask) + dispatcher := d.taskDispatcher() for ctx.Err() == nil { msg, err := d.Tasks.Fetch(ctx) if err != nil && !errors.Is(err, eventbus.ErrNoMessage) { @@ -1004,6 +1010,25 @@ func (d *Daemon) drainNATS(ctx context.Context) { dispatcher.Wait() } +// taskDispatcher returns the daemon's long-lived global dispatcher, building +// it from the running config on first use. It outlives a single drain pass so +// ResizeTaskDispatcher can resize the running dispatcher; drainNATS still +// waits for each pass's work before returning. +func (d *Daemon) taskDispatcher() *taskDispatcher { + d.dispatcherOnce.Do(func() { + d.dispatcher = newTaskDispatcher(d.Cfg.Get().Containers.MaxConcurrency, d.allowConcurrentForTask) + }) + return d.dispatcher +} + +// ResizeTaskDispatcher applies a new containers.max_concurrency to the +// running global task dispatcher without a restart. Running tasks keep their +// slots; a lowered limit is honoured as they finish, and a raised limit +// admits queued work immediately. +func (d *Daemon) ResizeTaskDispatcher(maxConcurrency int) { + d.taskDispatcher().SetMaxConcurrency(maxConcurrency) +} + // submitNATSTask decodes a fetched message and queues it for processing. A // message that cannot be decoded is acked rather than redelivered: it will // never decode on a retry, so leaving it unacked would block the queue. @@ -1026,27 +1051,55 @@ func (d *Daemon) submitNATSTask(ctx context.Context, dispatcher *taskDispatcher, // allowConcurrent reports true opt out of the same-repo serialization // entirely -- the global slot limit is the only bound on their concurrency. type taskDispatcher struct { - slots chan struct{} allowConcurrent func(task *workflow.Task) bool - mu sync.Mutex - repoTail map[string]chan struct{} - wg sync.WaitGroup + mu sync.Mutex + limit int + active int + slotsFree *sync.Cond + repoTail map[string]chan struct{} + wg sync.WaitGroup +} + +// SetMaxConcurrency accepts a new global limit without a restart. Running +// tasks keep their slots; a lowered limit is enforced as they finish, and a +// raised limit admits queued work immediately. +func (d *taskDispatcher) SetMaxConcurrency(maxConcurrency int) { + d.mu.Lock() + d.limit = maxConcurrency + d.mu.Unlock() + d.slotsFree.Broadcast() } func newTaskDispatcher(maxConcurrency int, allowConcurrent func(task *workflow.Task) bool) *taskDispatcher { - var slots chan struct{} - if maxConcurrency > 0 { - slots = make(chan struct{}, maxConcurrency) - } if allowConcurrent == nil { allowConcurrent = func(*workflow.Task) bool { return false } } - return &taskDispatcher{ - slots: slots, + d := &taskDispatcher{ + limit: maxConcurrency, allowConcurrent: allowConcurrent, repoTail: make(map[string]chan struct{}), } + d.slotsFree = sync.NewCond(&d.mu) + return d +} + +// acquireSlot blocks until the global limit has room and returns the release +// function. limit <= 0 means unlimited. The limit is re-read on every wake, so +// SetMaxConcurrency reaches waiting tasks without recreating the dispatcher. +func (d *taskDispatcher) acquireSlot() func() { + d.mu.Lock() + defer d.mu.Unlock() + for d.limit > 0 && d.active >= d.limit { + d.slotsFree.Wait() + } + d.active++ + return func() { + d.mu.Lock() + d.active-- + d.mu.Unlock() + d.slotsFree.Broadcast() + } } func (d *taskDispatcher) Submit( @@ -1069,10 +1122,8 @@ func (d *taskDispatcher) Submit( if previous != nil { <-previous } - if d.slots != nil { - d.slots <- struct{}{} - defer func() { <-d.slots }() - } + releaseSlot := d.acquireSlot() + defer releaseSlot() if done != nil { defer func() { close(done) diff --git a/internal/daemon/task_cancellation_test.go b/internal/daemon/task_cancellation_test.go index ee4e202f9..e3cc983b9 100644 --- a/internal/daemon/task_cancellation_test.go +++ b/internal/daemon/task_cancellation_test.go @@ -334,7 +334,7 @@ func TestProcessStopRunningTransitionsToParked(t *testing.T) { pool, err := archiecontainer.NewPool(ctx, archiecontainer.Config{ Image: "archie-agent:test", DockerClient: dockerClient, - }, d.Log) + }, nil, d.Log) if err != nil { t.Fatal(err) } diff --git a/internal/daemon/task_dispatcher_live_test.go b/internal/daemon/task_dispatcher_live_test.go new file mode 100644 index 000000000..00ddc4e55 --- /dev/null +++ b/internal/daemon/task_dispatcher_live_test.go @@ -0,0 +1,103 @@ +package daemon + +import ( + "context" + "log/slog" + "testing" + "time" + + "github.com/samcharles93/archie-core/internal/config" + "github.com/samcharles93/archie-core/internal/domain/workflow" +) + +// TestDaemonResizeTaskDispatcherTracksConfig pins the daemon's live-apply +// wrapper: the dispatcher is built from the running config and a later resize +// reaches it without rebuilding the daemon. +func TestDaemonResizeTaskDispatcherTracksConfig(t *testing.T) { + t.Parallel() + + d := &Daemon{ + Cfg: config.NewHolder(config.Config{Containers: config.ContainerConfig{MaxConcurrency: 2}}), + Log: slog.New(slog.DiscardHandler), + } + dispatcher := d.taskDispatcher() + dispatcher.mu.Lock() + booted := dispatcher.limit + dispatcher.mu.Unlock() + if booted != 2 { + t.Fatalf("dispatcher limit = %d, want the running config's 2", booted) + } + + d.ResizeTaskDispatcher(5) + dispatcher.mu.Lock() + resized := dispatcher.limit + dispatcher.mu.Unlock() + if resized != 5 { + t.Fatalf("dispatcher limit after resize = %d, want 5", resized) + } +} + +// TestTaskDispatcherResizesWithoutRestart pins the live half of +// archie-core-zfb0.2: the running dispatcher accepts a new +// containers.max_concurrency. Raising the limit admits queued work +// immediately; lowering it leaves running tasks holding their slots and +// admits new work only as they finish. +func TestTaskDispatcherResizesWithoutRestart(t *testing.T) { + t.Parallel() + + dispatcher := newTaskDispatcher(1, nil) + + releaseA := make(chan struct{}) + aStarted := make(chan struct{}) + bStarted := make(chan struct{}) + releaseB := make(chan struct{}) + cStarted := make(chan struct{}) + + dispatcher.Submit(context.Background(), &workflow.Task{Owner: "acme", Repo: "a"}, func(context.Context, *workflow.Task) { + close(aStarted) + <-releaseA + }) + <-aStarted + + dispatcher.Submit(context.Background(), &workflow.Task{Owner: "acme", Repo: "b"}, func(context.Context, *workflow.Task) { + close(bStarted) + <-releaseB + }) + select { + case <-bStarted: + t.Fatal("second task started above max_concurrency 1") + case <-time.After(50 * time.Millisecond): + } + + // Raising the limit must admit the queued task without a restart. + dispatcher.SetMaxConcurrency(2) + select { + case <-bStarted: + case <-time.After(time.Second): + t.Fatal("raising the limit did not admit the queued task") + } + + // Lowering the limit applies as running tasks finish: a new task waits + // while two tasks hold slots, and starts only once they have drained + // below the new limit. + dispatcher.SetMaxConcurrency(1) + dispatcher.Submit(context.Background(), &workflow.Task{Owner: "acme", Repo: "c"}, func(context.Context, *workflow.Task) { + close(cStarted) + }) + + close(releaseA) + select { + case <-cStarted: + t.Fatal("a task started while two tasks still held slots under a lowered limit") + case <-time.After(50 * time.Millisecond): + } + + close(releaseB) + select { + case <-cStarted: + case <-time.After(time.Second): + t.Fatal("lowered limit did not admit new work as running tasks finished") + } + + dispatcher.Wait() +} diff --git a/internal/daemon/terminal_cleanup_test.go b/internal/daemon/terminal_cleanup_test.go index 03d8856a4..9307e0de3 100644 --- a/internal/daemon/terminal_cleanup_test.go +++ b/internal/daemon/terminal_cleanup_test.go @@ -357,7 +357,7 @@ func TestProcessCleansWorktreeOnTerminalNoChange(t *testing.T) { pool, err := archiecontainer.NewPool(ctx, archiecontainer.Config{ Image: "archie-agent:test", DockerClient: dockerClient, - }, d.Log) + }, nil, d.Log) if err != nil { t.Fatal(err) }