From 024dfbbe49499dc83278fa351e8c05af59198d5b Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Tue, 22 Sep 2026 23:49:53 +1000 Subject: [PATCH 01/20] docs: delete ARCHITECTURE.md, move its normative content into docs/architecture ARCHITECTURE.md duplicated the docs/architecture tree and had drifted badly against it: "internal/ holds ~36" against 112 packages, two of the seven binaries listed, a Forge table listing `RepliesAfter` (deleted from the tree), and a stated invariant ("No daemon-internal cron") contradicted by the scheduling engine that internal/app/archied/scheduling.go composes and main.go starts. It was also the sole home of three pieces of still-authoritative content, so those move rather than disappear: - plugin engine rule (strict), the memory engine family, and the internal/plugin/architecture_test.go enforcement note -> docs/architecture/plugins-and-extensions.md - intake provenance: forge poll/webhook dedup through TaskEnvelope.IdempotencyKey, binding-stamped chat/playbook enqueue, and the single-identity webhook-intake rejection -> docs/architecture/messaging-and-work-intake.md Live references are repointed from CLAUDE.md (AGENTS.md is a symlink to it), docs/architecture, docs/guides, seven PRDs, and four .agents skills. The heading "## Plugin engine rule (strict)" is preserved in its new home so the existing #plugin-engine-rule-strict citations keep resolving. Generated docs artifacts regenerated. --- .agents/skills/archie-agent-fleet/SKILL.md | 2 +- .../archie-architecture-contract/SKILL.md | 14 +- .agents/skills/archie-build-and-env/SKILL.md | 2 +- .../skills/archie-docs-and-writing/SKILL.md | 5 +- ARCHITECTURE.md | 436 ------------------ CLAUDE.md | 15 +- docs/architecture/bindings.md | 9 +- .../architecture/messaging-and-work-intake.md | 26 ++ docs/architecture/plugins-and-extensions.md | 96 +++- docs/data/generated/dev-docs.json | 20 +- docs/data/generated/docs.json | 2 +- docs/guides/first-playbook.md | 2 +- docs/prds/deep-research-capability.md | 4 +- docs/prds/eda-playbook-engine.md | 3 +- docs/prds/event-sources-and-reactions.md | 3 +- docs/prds/image-capability-contract.md | 5 +- docs/prds/memory-engine-unification.md | 3 +- docs/prds/service-decomposition.md | 2 +- docs/prds/webhook-intake-security.md | 8 +- 19 files changed, 171 insertions(+), 486 deletions(-) delete mode 100644 ARCHITECTURE.md diff --git a/.agents/skills/archie-agent-fleet/SKILL.md b/.agents/skills/archie-agent-fleet/SKILL.md index f5df01062..9553b1003 100644 --- a/.agents/skills/archie-agent-fleet/SKILL.md +++ b/.agents/skills/archie-agent-fleet/SKILL.md @@ -158,7 +158,7 @@ Dolt remote is pushed explicitly (`bd dolt push`) and never assumed to be in syn `.pi/control-plane-run/` holds a run's scaffold, lane plan, prompts, scripts and gate output. It is gitignored and disposable. Treat it as run evidence only: a -fact worth keeping goes into this skill, `docs/`, or `ARCHITECTURE.md`, not into a +fact worth keeping goes into this skill or `docs/`, not into a run directory the next clean deletes. ## Worked example diff --git a/.agents/skills/archie-architecture-contract/SKILL.md b/.agents/skills/archie-architecture-contract/SKILL.md index b69c33121..40410e40f 100644 --- a/.agents/skills/archie-architecture-contract/SKILL.md +++ b/.agents/skills/archie-architecture-contract/SKILL.md @@ -63,13 +63,13 @@ Do not simplify these facts into the target model: | Workflow | `internal/domain/workflow` routes a `task.Task` into a `map[string]Workflow`, mutates a broad `TaskContext` through sequential stages. | `internal/domain/workflow/workflow.go` | | Agent-stage boundary | Versioned `agentexec.Request`/`Result` carries one autonomous stage; validation correlates task, attempt, and stage. | `internal/agentexec/protocol.go` | | Container handoff | `internal/taskrun` carries entire task plus `config.Repo` and `config.TaskConfig`; `internal/app/agentworker` owns routing and sequencing while its NATS adapter owns wire mechanics. | `internal/taskrun/taskrun.go`; `internal/app/agentworker/`; `internal/infrastructure/agenttransport/nats/` | -| Git ownership | Daemon-owned worktree operations clone, commit, push, diff, and clean up. | `internal/worktree/`; `internal/domain/workflow/steps.go`; `ARCHITECTURE.md` | +| Git ownership | Daemon-owned worktree operations clone, commit, push, diff, and clean up. | `internal/worktree/`; `internal/domain/workflow/steps.go`; `docs/architecture/agent-system.md` | | Enforcement | Gates, read-only/protected paths, TDD inverted test gates, and diff caps are represented outside prompt prose. | `internal/gate/`; `internal/domain/workflow/{agent,tdd,steps}.go` | | Configuration | `internal/config.Config`, `IdentityConfig`, `Repo`, and task snapshots mix input decoding with runtime concerns. Decoding itself lives in `internal/infrastructure/configuration`. | `internal/config/config.go`; `internal/infrastructure/configuration/` | | Identity | Configured names and bot usernames act as process-local identity keys. There is no durable Identity domain. | `internal/daemon/daemon.go`; `internal/domain/storecontract/storecontract.go`; `internal/gateway/session.go` | | Messaging | `internal/gateway` contains both the `Message` struct flow and richer `MessageEvent`; channel routing and task control are mixed with conversational behavior. | `internal/gateway/{gateway,messageevent,tasks}.go`; `internal/channels/` | | Plugins | `plugin.Plugin` exposes only `Name` and `Version`; current loading interprets operator-installed Go files with Yaegi. | `internal/plugin/plugin.go` | -| Processes | Agent modes include in-process, subprocess, and NATS. Optional containers add a distinct boundary with known gaps. | `internal/config/config.go`; `internal/app/archied/`; `ARCHITECTURE.md` | +| Processes | Agent modes include in-process, subprocess, and NATS. Optional containers add a distinct boundary with known gaps. | `internal/config/config.go`; `internal/app/archied/`; `docs/architecture/organisation.md` | ### Interpret the current lifecycle honestly @@ -98,13 +98,13 @@ not an open design. | Invariant | Required action | Status and source | | --- | --- | --- | -| The model never runs git | Keep clone, branch, commit, push, diff, and cleanup in deterministic workspace/worktree steps. | CURRENT: `ARCHITECTURE.md`; `internal/worktree/` | -| Deterministic constraints outrank prompts | Enforce gates, protected paths, read-only stages, test protection, and change caps outside model instructions. | CURRENT: `ARCHITECTURE.md`; `internal/gate/`; `internal/domain/workflow/` | -| Agent execution is a data boundary | Send bounded, versioned input; validate correlated output before applying it. | CURRENT foundation: `internal/agentexec/protocol.go`; `ARCHITECTURE.md` | +| The model never runs git | Keep clone, branch, commit, push, diff, and cleanup in deterministic workspace/worktree steps. | CURRENT: `docs/architecture/agent-system.md`; `internal/worktree/` | +| Deterministic constraints outrank prompts | Enforce gates, protected paths, read-only stages, test protection, and change caps outside model instructions. | CURRENT: `docs/architecture/agent-system.md`; `internal/gate/`; `internal/domain/workflow/` | +| Agent execution is a data boundary | Send bounded, versioned input; validate correlated output before applying it. | CURRENT foundation: `internal/agentexec/protocol.go`; `docs/architecture/agent-system.md` | | Every workflow ends explicitly | Preserve terminal outcome or visible park. Preserve crash recovery of interrupted `running` work. | CURRENT: `internal/domain/workflow/workflow.go`; `internal/store/store.go` | | Generic plugins stay metadata-only | Keep `plugin.Plugin` at `Name()`/`Version()`. Behavior on typed capability interface with owning `Registry`/`Manager`. | CURRENT-enforced rule: `internal/plugin/architecture_test.go` | -| Capability engines own lifecycle | Give resource-owning engines explicit start/health/stop semantics and failure isolation. | APPROVED rule: `ARCHITECTURE.md#plugin-engine-rule-strict` | -| Trust is explicit | Treat trusted operator-installed in-process code, repository code, out-of-process integrations, and container-isolated code as different trust classes. | APPROVED rule: `ARCHITECTURE.md#plugin-engine-rule-strict` | +| Capability engines own lifecycle | Give resource-owning engines explicit start/health/stop semantics and failure isolation. | APPROVED rule: `docs/architecture/plugins-and-extensions.md#plugin-engine-rule-strict` | +| Trust is explicit | Treat trusted operator-installed in-process code, repository code, out-of-process integrations, and container-isolated code as different trust classes. | APPROVED rule: `docs/architecture/plugins-and-extensions.md#plugin-engine-rule-strict` | | Infrastructure stays replaceable | Keep forge, persistence, workspace, transport, container, and model details behind behavior-owned contracts. | APPROVED TARGET: `docs/architecture/dependencies-and-contracts.md` | | Same-repository execution serializes by default | Preserve default unless explicit scheduling policy permits concurrency. | CURRENT: `internal/config/config.go`; `internal/daemon/daemon.go` | diff --git a/.agents/skills/archie-build-and-env/SKILL.md b/.agents/skills/archie-build-and-env/SKILL.md index 2e4053818..d329d3d78 100644 --- a/.agents/skills/archie-build-and-env/SKILL.md +++ b/.agents/skills/archie-build-and-env/SKILL.md @@ -26,7 +26,7 @@ No root `README*` or `CONTRIBUTING*` file exists. Begin with: ```bash sed -n '1,240p' CLAUDE.md -sed -n '1,240p' ARCHITECTURE.md +ls docs/architecture/ task --list ``` diff --git a/.agents/skills/archie-docs-and-writing/SKILL.md b/.agents/skills/archie-docs-and-writing/SKILL.md index f7b17c995..b83bb71cd 100644 --- a/.agents/skills/archie-docs-and-writing/SKILL.md +++ b/.agents/skills/archie-docs-and-writing/SKILL.md @@ -1,6 +1,6 @@ --- name: archie-docs-and-writing -description: Maintain Archie's documentation of record and write durable architecture decisions, migration plans, parity matrices, incident and dead-end records, feature ownership and deprecation records, operational runbooks, and documentation reviews. Use when deciding where an Archie fact belongs; reconciling code with ARCHITECTURE.md, CLAUDE.md, docs/prds, docs/archive, or generated contracts; changing tools/docsgen; documenting a feature so future maintainers can find its owners, consumers, invariants, superseded paths, deletion gates, evidence, and rollback; or checking documentation authority, drift, links, copied content, and source-artifact hygiene. +description: Maintain Archie's documentation of record and write durable architecture decisions, migration plans, parity matrices, incident and dead-end records, feature ownership and deprecation records, operational runbooks, and documentation reviews. Use when deciding where an Archie fact belongs; reconciling code with `CLAUDE.md`, `docs/architecture`, `docs/prds`, `docs/archive`, or generated contracts; changing `tools/docsgen`; documenting a feature so future maintainers can find its owners, consumers, invariants, superseded paths, deletion gates, evidence, and rollback; or checking documentation authority, drift, links, copied content, and source-artifact hygiene. --- # Maintain Archie documentation @@ -42,7 +42,6 @@ Verified on 2026-09-18: |---|---|---| | Live Go code, tests, `Taskfile.yml`, executable config parsing, composition under `internal/app/` (entered from `cmd/`) | `CURRENT` execution evidence | Trace producers and consumers; tests prove only asserted behavior | | `CLAUDE.md` | Current contributor and safety protocol plus compact architecture orientation | Verify operational claims against code and `Taskfile.yml`; `AGENTS.md` is a symlink to it | -| `ARCHITECTURE.md` | Useful architecture history and partial current overview | Corroborate every inventory/status claim; its "Planned" list is stale for implemented skill support | | `docs/prds/01-project-management.md` | Index and approved foundation for target architecture | Add or change target decisions in the focused document it names | | `docs/architecture/*.md` | Focused target decisions, active review procedure, migration inventory | Read each file's status | | `docs/architecture/migration-decisions.md` | `OPEN` migration inventory constrained by approved decisions | Close a question only after code-grounded review | @@ -77,7 +76,7 @@ The repository root has no `README.md` or `CONTRIBUTING.md` as of 2026-09-18. | Superseded design retained for context | `docs/archive/`, with historical status | | Incident or rejected approach | `archie-failure-archaeology`'s chronology | -Do not duplicate a package map across `CLAUDE.md`, `ARCHITECTURE.md`, a PRD, and +Do not duplicate a package map across `CLAUDE.md`, a PRD, and the site. Name the authoritative map and link to it. ## Write for future change diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md deleted file mode 100644 index d23de5d2f..000000000 --- a/ARCHITECTURE.md +++ /dev/null @@ -1,436 +0,0 @@ -# archie-core Architecture - -**archied** is a resident daemon that discovers forge work — GitHub and Gitea -issues assigned to its bot user, carrying its label, or either (`dispatch` -mode, per identity) — by polling (the default) or, single-identity -deployments only, reacting to forge webhook events the instant they arrive -(`Forge.Intake` = `webhook`/`both`, see `docs/prds/event-sources-and- -reactions.md`), works each matched issue in an isolated worktree through a -routed workflow, and opens pull requests for human review. A single daemon can -serve several identities across several forges, each with its own `bot_user`, -label, and repository set. - -## Package Map - -This lists the load-bearing packages, not all of them — `internal/` holds ~36. -The domain migration is in progress (`internal/domain/`, -`internal/infrastructure/`), so **the tree is authoritative over this table**. -See `docs/architecture/organisation.md` for the target structure and -`docs/architecture/migration-decisions.md` for the current position. - -| Package | Role | -| ---------------------------- | ------------------------------------------------------------------------- | -| `cmd/archied/` | Binary entry: config→forge→store→runtime→workflows→daemon | -| `cmd/archie-agent/` | Binary entry: flags, environment, signals, and agentworker invocation | -| `internal/app/agentworker/` | Agent worker composition, lifecycle, routing, and workflow execution | -| `internal/infrastructure/agentboot/` | Worker boot-brief filesystem decoding | -| `internal/infrastructure/agentgit/` | Worker git safe-directory setup | -| `internal/infrastructure/agenttransport/nats/` | Worker core-NATS full-task handoff and RPC client construction | -| `internal/config/` | Config types only — loading lives in `infrastructure/configuration` | -| `internal/daemon/` | Resident loop: poll, enqueue, claim, route, process, reconcile | -| `internal/domain/workflow/` | Engine: stage lists, routing, shared steps, workflow definitions | -| `internal/domain/workflow/task/` | Task-execution vocabulary (`Task`, statuses, mid-run `Store`, `Definition`) importable without the engine | -| `internal/domain/messaging/` | Canonical messages plus the chat wire contracts (`ChatContract`, sessions, turns, streaming) shared by gateway, channels, webui, and archie-ui | -| `internal/channels/status/` | Operator-facing channel lifecycle ledger (`State`/`Descriptor`/`Status`/`Manager`); the plugin contract stays in `internal/channels` | -| `internal/app/archiemessaging/` | Messaging Service composition: Telegram/email/webhook channels, dialling the Gateway's `ChatContract` over gRPC. The daemon runs no channel and builds no `gateway.Router` | -| `internal/domain/workintake/`| `TaskEnvelope`, routing `Kind`, label vocabulary, task subjects | -| `internal/agentexec/` | Worker-local stage protocol and ai-sdk loop runner | -| `internal/forge/` | Forge interface: GitHub and Gitea implementations | -| `internal/forge/webhook/` | Optional forge webhook receiver: GitHub issue events → `workintake.TaskEnvelope`, same publish path as polling | -| `internal/webhookguard/` | Cross-cutting inbound-webhook mechanics: HMAC verification, per-source rate limiting | -| `internal/worktree/` | Git operations: clone, branch, commit, push, diff, cleanup | -| `internal/store/` | Task/capture/mapping/binding SQLite implementation; producer-owned contract interfaces live in `internal/domain/storecontract` (aliases kept here) | -| `internal/domain/storecontract/` | Producer-owned State Store read contracts consumed by webui, intake, and the gRPC adapters (state-store-contract rev. 2d) | -| `internal/events/` | In-process event bus: publish/subscribe for observability | -| `internal/infrastructure/` | Config loading, eventbus transport, model catalog | -| `internal/webui/` | Shared dashboard HTTP layer served by the archie-ui process | -| `internal/gateway/` (18.7k LOC) | Persistent-connection layer between archie and its users (Telegram, web UI, etc.); shared CommandRouter for dispatch | -| `internal/tools/` (20.9k LOC) | Central tool subsystem: built-in tool implementations, approval, execution contract | -| `internal/domain/memory/` | Memory engine contract: four typed scopes, CRUD + revisions, content scanner | -| `internal/infrastructure/memory/` | Builtin memory engine + its `builtin/` markdown store | -| `internal/secret/` | Secret reference resolution across named engines (env, vault, etc.) | -| `internal/plugin/` | Core plugin interface: Yaegi-interpreted `.go` plugins loaded from `~/.config/archie/plugins/` and registered with the daemon's extension registry | -| `internal/skill/` | Parses and discovers agentskills.io `SKILL.md` files | -| `ai-sdk/runtime` | External: provider-agnostic LLM runtime | -| `ai-sdk/agentloop` | External: gated agent loop with budgets and tool jails | -| `ai-sdk/core` | External: tool definitions, generate options | - -## Task Lifecycle - -``` -queued → running(workflow:stage) → pr_open → merged | rejected - ↓ - waiting_human → (human approves) → queued(implement) - ↓ - parked -``` - -**Crash recovery:** tasks left in `running` are re-queued on startup. Parks are -never silent, but they no longer comment on the forge issue: the reason is -recorded on the task as `park_reason` (cleared on requeue) and surfaced by the -dashboard and `/api/tasks`. Retries are an explicit operator action capped by -`max_retries`, not a label a human removes. - -**Adversarial review:** the implement workflow runs a fresh adversarial review -after the deterministic gates and before `pr_open` (see -`docs/architecture/adversarial-review.md`). A surviving blocking finding — or a -review that failed to run — parks the task with the findings as its -`park_reason`; otherwise the PR opens with a findings section on its body. -Opt-in per repo via `repo.review_enabled`. - -**PR reconciliation:** the daemon polls open PRs, checks GitHub state -(`merged`/`closed`), transitions the task, and cleans up worktrees. - -**Provenance by origin:** a task's origin determines which enqueue path -created it, though every origin converges on the same `tasks` table and the -same `Route`/`Run` lifecycle above once queued. Forge-sourced tasks — whether -discovered by polling or, single-identity only, delivered by the forge -webhook receiver (`internal/forge/webhook`) — enqueue via the same -`workintake.TaskEnvelope` and `PublishTask` path (see `docs/prds/event-sources- -and-reactions.md`'s decided contract); `TaskEnvelope.IdempotencyKey` is keyed -on `owner/repo/number`, not delivery source, so a webhook-delivered issue and -a later poll of the same issue dedup for free instead of double-enqueuing. -Playbook-binding-dispatched tasks enqueue -via `EnqueueChatTask`/`EnqueueBindingTask` (`internal/store`), the same direct -path chat-spawned tasks use, stamped with `binding_id`/`binding_version` for -provenance. See `docs/architecture/bindings.md` for the binding model and its -threat model in full. - -## Workflow Engine - -### Core types (`workflow.go`) - -- `Workflow` = named list of `Stage`s -- `Stage` = `Name` + `Run(ctx, *TaskContext) error` -- `TaskContext` = mutable bag: Task, Repo, Config, Forge, Store, Agent runner, - worktree dir/branch, scratch fields -- `Outcome` = terminal decision (status + detail) -- ends the workflow - immediately -- `Registry` = `map[string]Workflow` - -### Routing (`Route`) - -Label-driven, no LLM involved. Pre-assigned workflow wins; then labels decide. -The built-in bindings are: - -- `bug` → `tdd` -- `feature` → `feasibility` -- `bootstrap` → `bootstrap` (diagnostics, no LLM spend) -- Default → `implement` - -Both the closed `bug`/`feature`/`bootstrap` set's target workflow and -arbitrary additional labels are YAML-overridable at daemon startup -(`internal/domain/workflow/routing.go`'s `LoadKindWorkflowsYAML`/ -`LoadLabelWorkflowsYAML`, `WorkflowRoutingFile`/`WorkflowLabelsFile`/ -`PlaybookDirs` config) -- see `docs/prds/eda-playbook-engine.md`. A label -already owned by the closed kind set, or a binding declared twice across -sources, is a reported load failure, not a silent override. - -### Execution (`Run`) - -Stages execute sequentially. After each stage: - -- Task is persisted to SQLite -- Error → park the task (comment on issue, keep worktree) -- Outcome set → finish (transition task, update labels) -- Neither → continue to next stage - -A workflow that ends without an outcome parks -- definition bugs must not vanish -silently. - -## Workflows - -### Bootstrap (diagnostics, no LLM) - -``` -prepare → apply(marker file) → commit-push → diff-cap → open-pr -``` - -Proves the plumbing: clone, push, PR. Label any test issue `bootstrap` to verify -end-to-end. - -### Implement (default) - -``` -prepare → baseline-gate → plan(planner, read-only) → build(builder, gated) → commit-push → diff-cap → open-pr -``` - -- **baseline-gate:** runs the repo's gate at the base commit before anything is - touched -- **plan:** read-only agent explores codebase, produces implementation plan, - posts it as a comment -- **build:** full toolset agent implements against the plan behind the repo's - quality gate -- **diff-cap:** parks oversized changes (>400 lines default) -- **open-pr:** builder's summary becomes the PR body - -### TDD (bug label) - -``` -prepare → baseline-gate → analyse(planner) → repro-tests(builder, inverted gate) → capture-proof → commit-repro → fix(builder, normal gate, test files protected) → commit-push → diff-cap → open-pr + repro-evidence -``` - -- **repro-tests:** the quality gate is **inverted** -- the test command must - FAIL. Proven failure is required. -- **capture-proof:** runs the test command deterministically, captures the - failing output -- **commit-repro:** commits the failing tests as the first commit -- **fix:** test files are **environmentally write-protected** (not - prompt-ruled). The agent fixes code, not tests. -- **open-pr + evidence:** posts both the PR and the captured failure proof - -### Feasibility (feature label) - -``` -prepare → assess(planner, with decide tool) → [close_if_bad_fit | prd(planner)] → deliver → waiting_human -``` - -- **assess:** reads AGENT.md, ROADMAP.md, README. Calls a `decide` tool - (structured output, not parsed prose). Bad fit → close with reasons. -- **prd:** read-only agent writes a PRD: problem, solution, files affected, - acceptance criteria, non-goals, diff estimate -- **deliver:** posts PRD as issue comment + webhook notification (n8n → email). - Sets state to `waiting_human`. -- The daemon watches for human replies, judges them with a triage LLM - (APPROVE/REJECT/UNCLEAR), requeues or closes. - -## Forge Interface - -A thin abstraction over the forge. GitHub (`github.go`) and Gitea (`gitea.go`) -both implement it, plus a no-op (`noop.go`) used when no credential resolves — -see the degradation rule in `docs/architecture/configuration.md`. A multi-forge -deployment is configured per identity; see -`deployments/multi-forge-github-gitea.toml`. - -| Method | Purpose | -| ------------------- | --------------------------------------- | -| `AcceptInvitations` | Auto-accept repo invites | -| `AssignedIssues` | Poll assigned issues (excluding PRs) | -| `IssuesWithLabel` | Poll labelled issues | -| `Comment` | Post issue/PR comment, return ID | -| `RepliesAfter` | Find human replies after a comment ID | -| `CreatePR` | Open pull request | -| `PRState` | Check merged/open/closed | -| `CloseIssue` | Close with final comment | -| `React` | Emoji reaction (pickup acknowledgement) | -| `SetStateLabel` | Replace old state label with new one | -| `VerifyPush` | Confirm push permission at startup | - -State labels (`archie:queued`, `archie:working`, etc.) are created on demand, -removed when transitioning. They report state; they do not drive it. Removing -a label no longer retries a parked task -- retries are an explicit operator -action from the dashboard or chat, capped by `max_retries`. - -### Webhook intake (alternative to polling) - -`internal/forge/webhook` is a separate HTTP receiver, not a `Forge` method: it -verifies a GitHub webhook's HMAC signature (`internal/webhookguard.VerifyHMAC`), -decodes an `issues` event, applies the same dispatch predicate the poller -uses, and calls the same `PublishTask` the poller calls. It is opt-in via -`Forge.Intake` (`poll` default, `webhook`, `both`) and single-identity only: a -config naming `webhook`/`both` together with `[[identities]]`, or setting -intake on an identity, is rejected at validation rather than started with the -setting ignored -- the receiver binds one secret and one dispatch predicate, -so it cannot judge a second identity's events by the right rules. A -multi-identity deployment polls per identity until per-identity intake lands. -See `docs/prds/event-sources-and-reactions.md` for why this stayed a -thin decode-and-reuse layer rather than a second intake mechanism. - -## Worktree Manager - -Every task gets a fresh clone -- no persistent state between runs. The model -never runs git. - -``` -~/.local/share/archie/work/-/issue-/ -``` - -Token auth via a generated `GIT_ASKPASS` helper script -- the token never -appears in `.git/config` or process argv. - -## Config - -Daemon-level TOML (`~/.config/archie/config.toml`): - -- `[forge]` -- type (github/gitea), host, token secret reference; `intake` - (`poll` default, `webhook`, or `both`) with `webhook_secret`/`webhook_addr` - when reacting to forge events instead of, or alongside, polling - (single-identity deployments only; rejected, not ignored, alongside - `[[identities]]` or on a `[identities.forge]` entry) -- `[dispatch]` -- trigger (assignee/label/either), labels, ack reaction -- `[[repos]]` -- owner, name, base branch, gate commands, protected paths, - ecosystem -- `[models]` -- planner, builder, triage (provider/model refs) -- `[providers]` -- LLM provider configs -- `[nats]` -- embedded or external broker deployment -- `[containers]` -- managed `archie-agent` image, limits, storage, and network -- `[budgets]` -- max steps, max tokens, wall clock, gate max failures -- `[web]` -- dashboard listen address, bound by the standalone archie-ui - process (the daemon runs no dashboard listener after the UI cutover) -- `[notify]` -- webhook URL for notifications - -Secrets (forge token, provider API keys, channel tokens) are not embedded in -config files. `internal/secret` defines a pluggable `Engine` interface (`Name`, -`Version`, `Resolve(key) (string, error)`) resolved through a `Registry`; the -built-in `env` engine reads an environment variable by name and the built-in -`bws` engine caches the accessible Bitwarden project at startup. Custom engines -(sops, Vault/OpenBao) are Yaegi-interpreted `.go` plugins loaded from a -directory, following the same convention as `internal/plugin`'s daemon plugins. -Config fields reference a secret as -`{engine, key}` (`secret.SecretRef`), e.g. -`token = { engine = "env", key = "ARCHIE_GITHUB_TOKEN" }`. As of this writing -`Forge.Token` and `Provider.APIKey` use `SecretRef`; resolved provider values -are exported under private, identity-scoped environment names because the SDK -and container boundary consume credential names rather than -plaintext wire fields. The older `Provider.APIKeyEnv`, `NATSConfig.TokenEnv`, and -`TelegramConfig.TokenEnv` field names remain supported. - -Per-repo gates are command lists: `[[repos.gate]] = ["go", "vet", "./..."]`. The -last command is the test runner (by convention -- TDD inverts it). - -## Observability - -Every lifecycle event flows through an internal event bus: - -- SQLite event log (every event stamped with a row ID) -- Live dashboard via SSE (server-sent events) -- Bounded subscriber buffers -- slow consumers drop events, never block the - daemon - -## Plugin engine rule (strict) - -“Engine” is reserved for a typed, lifecycle-managed capability family. It is not -a synonym for a generic plugin hook. Every new plugin engine must satisfy all of -these invariants: - -1. **Typed domain contract.** The engine interface exposes real - capability-specific operations in addition to identity metadata. Examples are - resolving a secret, delivering a message, or running an agent. A plugin that - only exposes `Name` and `Version` is metadata, not an engine. -2. **Owning family manager.** The capability package owns a `Registry` or - `Manager` that controls provider registration and discovery, initialization, - or access. Providers do not register arbitrary callbacks directly on the - daemon. -3. **Lifecycle and health.** When an implementation owns resources or background - work, its typed family contract must expose start, health, and stop - semantics. The owning manager invokes them and defines failure isolation and - shutdown ordering. Stateless providers may make these operations explicit - no-ops. -4. **Narrow host access.** Plugins receive the typed family registrar and only - the host services their declared capability requires. They never receive - `*daemon.Daemon`, a service locator, or an untyped hook map. -5. **Metadata-only generic plugin contract.** `plugin.Plugin` remains limited to - `Name()` and `Version()`. Domain hooks belong on typed engine interfaces; - adding capability methods to the generic interface is forbidden. -6. **Trust boundary stays explicit.** Explicitly trusted, operator-installed - plugins—including third-party secret-engine plugins the operator chooses to - trust—may run in-process and have daemon privileges. Repository-supplied code - runs in the task container. Integrations that are not trusted as daemon code - run out of process through MCP or a versioned protocol and, when a - confidentiality or integrity boundary is required, inside a real container - sandbox. A subprocess boundary alone is not a security sandbox, and calling - Yaegi-loaded code a plugin does not isolate it. - -`internal/plugin/architecture_test.go` enforces the mechanically testable parts -of this rule: the generic plugin method set, the shared agent-instruction -source, and the requirement that engine interfaces have domain behavior plus an -owning registry or manager. - -### Memory engine family - -One engine, addressed by four typed scopes -- `ScopeGlobal`, `ScopeAgent`, -`ScopeUser`, `ScopeAgentUser` (`internal/domain/memory`, `Scope`/`ScopeKind`). -A caller names exactly the scopes it may read or write; the engine applies no -access policy of its own, so isolation falls out mechanically because -`ScopeAgent{a}` and `ScopeAgent{b}` are different storage keys. See -`docs/prds/memory-engine-unification.md` for the decision this contract -implements; it superseded `internal/memory` (`MemoryProvider`/`Manager`), -deleted in that PRD's slice 5. - -`MemoryEngine` (`internal/domain/memory/contract.go`) is the typed family -contract: `Create`, `Get`, `Query`, `List`, `Update`, `Forget`, `Revisions` -- -CRUD with revisions retained, not just create/read/delete. `Update` supersedes -rather than overwrites: the superseded state is appended to the scope's -`HISTORY.md` before the live block is replaced, and `Update` carries no -`Scope` field, so a record can never change scope. The family's content -scanner (prompt-injection / sensitive-data patterns) is applied inside -`Create` and `Update`, the single choke point every producer crosses. - -`internal/domain/memory.Registry` is the owning family registry: registration, -start/health/stop, shutdown ordering, and failure isolation, following the -plugin engine rule (`ARCHITECTURE.md#plugin-engine-rule-strict`). -`internal/infrastructure/memory.NewBuiltinEngine` is the only registered -engine today; it persists through `internal/infrastructure/memory/builtin`, -a markdown store (`MEMORY.md`-shaped per-scope files) that owns its on-disk -format end-to-end. - -Two per-turn seams in `internal/gateway` connect a chat turn to the engine, -each narrowed to exactly the operations it needs: - -- **Read** (`turn_memory.go`, `MemoryStore`): `prepareTurn` runs a - scope-only `Query` over the resolved `Subject`'s readable scopes, - synchronously and before the prompt is built. It never returns an error -- - a panicking or failing engine degrades to an empty `` block rather - than failing the turn -- and is bounded by a per-scope record limit and a - byte cap on the rendered block. -- **Write** (`turn_memory_tool.go`, `MemoryWriteStore`): four tools -- - `memory_create`, `memory_update`, `memory_delete`, `memory_list` -- built - per turn onto the resolved `Subject`'s writable scopes, not registered once - at boot. The model chooses the scope kind; it never chooses the agent or - user id, which come from the turn's resolved `Subject` and have no schema - field to override. `ScopeGlobal` is never model-writable. - -`boot.setupMemoryEngine` (`internal/app/archied/bootstrap.go`) registers and -starts the engine and must run before `setupGatewayChat`, which constructs the -only chat turn runner in production and captures the engine at construction -time (`internal/app/archied/composition_order_test.go` pins the ordering), and -before `setupCurators`, whose `Registrar` captures the engine registry. The session curator (`internal/infrastructure/sessioncurator`) -writes derived records onto `ScopeAgentUser`, making it the engine's other -live producer besides the chat write path. - -## Key Design Decisions - -1. **Environmental constraints over prompt rules.** The gate, test-file - protection, and diff cap are code enforcement -- not "please don't do that" - in the system prompt. -2. **The model never runs git.** Worktree operations are deterministic steps, - not agent tools. -3. **Tokens as cost, not features.** Token usage is tracked per task and emitted - in observability events but never budgeted beyond the agent loop's own - limits. -4. **Label-driven routing.** Workflow selection is a switch statement, not an - LLM call. A triage LLM is used only for the human reply judgment in - waiting_human. -5. **Shared code.** `ai-sdk/agentloop` and `ai-sdk/toolkit` provide the common - agent loop and tool set. -6. **No daemon-internal cron.** Polling is a ticker loop, not cron. The - feasibility PRD notification uses an external webhook (n8n). -7. **Fresh worktree per task.** No persistent state between runs. Crash recovery - just re-queues. -8. **Agent execution is a data boundary.** Autonomous stages receive a versioned - request and return a fenced result; SQLite, forge operations, git, workflow - sequencing, and outcome handling stay daemon-owned. - -## What's Implemented vs Planned - -**Implemented:** - -- Daemon poll loop, enqueue, claim, process -- Forge webhook intake as a feature-flagged alternative/backstop pairing to - polling (single-identity deployments; see `internal/forge/webhook`) -- Bootstrap, implement, TDD, feasibility workflows -- GitHub and Gitea forges (issues, PRs, labels, comments, reactions) -- Worktree isolation (clone, branch, commit, push, diff, cleanup) -- SQLite store with transition audit -- Config with ecosystem support (go, python, node, rust) -- Agent stages with gates, budgets, preflight, protected paths -- Observability (event bus, SQLite log, SSE dashboard) -- Crash recovery (re-queue stale running tasks) -- Human reply judgment (triage LLM for waiting_human) - -**Planned:** - -- Multi-identity webhook intake (currently single-identity only) -- More ecosystems -- Notification integrations beyond n8n webhook -- Skills (SKILL.md support -- see `.archie/skills/`) diff --git a/CLAUDE.md b/CLAUDE.md index 2b2b912ec..119e06999 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -25,11 +25,13 @@ assigned/labelled to it, works each one in an isolated git worktree through a routed workflow (bootstrap / implement / TDD / feasibility), and opens pull requests for human review. -See `ARCHITECTURE.md` for the package map, workflow engine design, task -lifecycle, and key design decisions. Read it before making non-trivial changes. -Key invariants: env-enforced gates, model never runs git, and agent execution -functions as a strict data boundary. Before adding a new plugin engine, satisfy -`ARCHITECTURE.md#plugin-engine-rule-strict`. +Architecture of record lives in `docs/architecture/` -- `organisation.md` for +the package map, `agent-system.md` for the workflow-execution lifecycle and its +invariants, `plugins-and-extensions.md` for engine families. Read it before +making non-trivial changes. Key invariants: env-enforced gates, model never runs +git, and agent execution functions as a strict data boundary. Before adding a new +plugin engine, satisfy +`docs/architecture/plugins-and-extensions.md#plugin-engine-rule-strict`. ## Scope Discipline @@ -257,7 +259,8 @@ structures found in legacy packages. - Webhook intake refuses to start when `[[identities]]` is configured (one receiver, one dispatch config). Do not silently enable it for multi-identity deployments without first building per-identity routing -- see - `ARCHITECTURE.md`'s "Webhook intake" section. + `docs/architecture/messaging-and-work-intake.md`'s "Intake provenance" + section. - **`internal/domain/embedding/` + `internal/infrastructure/embedding/`:** Embedding client contract + implementation. Config-driven exactly like chat model roles: `models.embedding = "provider/model"` plus a `[providers.*]` diff --git a/docs/architecture/bindings.md b/docs/architecture/bindings.md index 188410ef7..07dc3b78f 100644 --- a/docs/architecture/bindings.md +++ b/docs/architecture/bindings.md @@ -5,9 +5,9 @@ through `.5`, `.7`, `.9`-`.15`) **Date:** 2026-09-03 This document covers the binding model and the intake threat model portion -of `archie-core-t2db.6`. It does not cover the `ARCHITECTURE.md` task-lifecycle -addendum or the operator walkthrough -- those remain separate, open pieces of -the same ticket. +of `archie-core-t2db.6`. It does not cover the task-lifecycle provenance rule +(`messaging-and-work-intake.md`'s "Intake provenance") or the operator +walkthrough -- that walkthrough remains an open piece of the same ticket. ## What a binding is @@ -177,9 +177,6 @@ execution-time idempotency, now landed (`archie-core-t2db.17` closed): the ## What this document does not cover -- The `ARCHITECTURE.md` task-lifecycle section noting that a - playbook-originated task has different provenance from a forge-issue - task -- open, part of `archie-core-t2db.6`. - An end-to-end operator walkthrough (point an external app at archie, build a first playbook) -- open, part of `archie-core-t2db.6`. - The playbook YAML / CEL / Module/Channel/Forge action-position work diff --git a/docs/architecture/messaging-and-work-intake.md b/docs/architecture/messaging-and-work-intake.md index 22971a809..d7f6b8c93 100644 --- a/docs/architecture/messaging-and-work-intake.md +++ b/docs/architecture/messaging-and-work-intake.md @@ -251,6 +251,32 @@ Current intake is fragmented: - `store.Task` collapses an external source reference and durable execution state. +## Intake provenance + +A task's origin determines which enqueue path created it, though every origin +converges on the same `tasks` table and the same `Route`/`Run` lifecycle once +queued. + +- Forge-sourced work -- discovered by polling or, in single-identity + deployments, delivered by the forge webhook receiver (`internal/forge/webhook`) + -- enqueues through the same `workintake.TaskEnvelope` and `PublishTask` path. + `TaskEnvelope.IdempotencyKey` is keyed on `owner/repo/number` and not on the + delivery source, so a webhook-delivered issue and a later poll of the same + issue dedup through `PublishUnique` instead of double-enqueuing. +- Chat-spawned and playbook-binding-dispatched work enqueues through + `EnqueueChatTask`/`EnqueueBindingTask` (`internal/store`), stamped with + `binding_id`/`binding_version` for provenance. + +Webhook intake is opt-in per deployment (`Forge.Intake` = `poll` (default), +`webhook`, or `both`) and single-identity only: a config naming +`webhook`/`both` alongside `[[identities]]`, or setting intake on a +`[identities.forge]` entry, is rejected at validation rather than started with +the setting ignored. One receiver binds one secret and one dispatch predicate, +so it cannot judge a second identity's events by the right rules. A +multi-identity deployment polls per identity until per-identity intake lands. +The decided contract is `docs/prds/event-sources-and-reactions.md`; the binding +model and its threat model are `docs/architecture/bindings.md`. + ## Current hazards - Treating every agent interaction as a work request would remove Archie's diff --git a/docs/architecture/plugins-and-extensions.md b/docs/architecture/plugins-and-extensions.md index 58c8dfcec..1374a1888 100644 --- a/docs/architecture/plugins-and-extensions.md +++ b/docs/architecture/plugins-and-extensions.md @@ -23,6 +23,97 @@ versioning, definition, and execution contracts. The generic plugin domain may provide plugin identity, discovery, compatibility, and lifecycle mechanics, but it MUST NOT define Workflow semantics or expose untyped Workflow hooks. +## Plugin engine rule (strict) + +“Engine” is reserved for a typed, lifecycle-managed capability family. It is not +a synonym for a generic plugin hook. Every new plugin engine must satisfy all of +these invariants: + +1. **Typed domain contract.** The engine interface exposes real + capability-specific operations in addition to identity metadata. Examples are + resolving a secret, delivering a message, or running an agent. A plugin that + only exposes `Name` and `Version` is metadata, not an engine. +2. **Owning family manager.** The capability package owns a `Registry` or + `Manager` that controls provider registration and discovery, initialization, + or access. Providers do not register arbitrary callbacks directly on the + daemon. +3. **Lifecycle and health.** When an implementation owns resources or background + work, its typed family contract must expose start, health, and stop + semantics. The owning manager invokes them and defines failure isolation and + shutdown ordering. Stateless providers may make these operations explicit + no-ops. +4. **Narrow host access.** Plugins receive the typed family registrar and only + the host services their declared capability requires. They never receive + `*daemon.Daemon`, a service locator, or an untyped hook map. +5. **Metadata-only generic plugin contract.** `plugin.Plugin` remains limited to + `Name()` and `Version()`. Domain hooks belong on typed engine interfaces; + adding capability methods to the generic interface is forbidden. +6. **Trust boundary stays explicit.** Explicitly trusted, operator-installed + plugins—including third-party secret-engine plugins the operator chooses to + trust—may run in-process and have daemon privileges. Repository-supplied code + runs in the task container. Integrations that are not trusted as daemon code + run out of process through MCP or a versioned protocol and, when a + confidentiality or integrity boundary is required, inside a real container + sandbox. A subprocess boundary alone is not a security sandbox, and calling + Yaegi-loaded code a plugin does not isolate it. + +`internal/plugin/architecture_test.go` enforces the mechanically testable parts +of this rule: the generic plugin method set, the shared agent-instruction +source, and the requirement that engine interfaces have domain behavior plus an +owning registry or manager. + +### Memory engine family + +One engine, addressed by four typed scopes -- `ScopeGlobal`, `ScopeAgent`, +`ScopeUser`, `ScopeAgentUser` (`internal/domain/memory`, `Scope`/`ScopeKind`). +A caller names exactly the scopes it may read or write; the engine applies no +access policy of its own, so isolation falls out mechanically because +`ScopeAgent{a}` and `ScopeAgent{b}` are different storage keys. See +`docs/prds/memory-engine-unification.md` for the decision this contract +implements; it superseded `internal/memory` (`MemoryProvider`/`Manager`), +deleted in that PRD's slice 5. + +`MemoryEngine` (`internal/domain/memory/contract.go`) is the typed family +contract: `Create`, `Get`, `Query`, `List`, `Update`, `Forget`, `Revisions` -- +CRUD with revisions retained, not just create/read/delete. `Update` supersedes +rather than overwrites: the superseded state is appended to the scope's +`HISTORY.md` before the live block is replaced, and `Update` carries no +`Scope` field, so a record can never change scope. The family's content +scanner (prompt-injection / sensitive-data patterns) is applied inside +`Create` and `Update`, the single choke point every producer crosses. + +`internal/domain/memory.Registry` is the owning family registry: registration, +start/health/stop, shutdown ordering, and failure isolation, following the +plugin engine rule above. `internal/infrastructure/memory.NewBuiltinEngine` is +the only registered engine today; it persists through +`internal/infrastructure/memory/builtin`, a markdown store (`MEMORY.md`-shaped +per-scope files) that owns its on-disk format end-to-end. + +Two per-turn seams in `internal/gateway` connect a chat turn to the engine, +each narrowed to exactly the operations it needs: + +- **Read** (`turn_memory.go`, `MemoryStore`): `prepareTurn` runs a + scope-only `Query` over the resolved `Subject`'s readable scopes, + synchronously and before the prompt is built. It never returns an error -- + a panicking or failing engine degrades to an empty `` block rather + than failing the turn -- and is bounded by a per-scope record limit and a + byte cap on the rendered block. +- **Write** (`turn_memory_tool.go`, `MemoryWriteStore`): four tools -- + `memory_create`, `memory_update`, `memory_delete`, `memory_list` -- built + per turn onto the resolved `Subject`'s writable scopes, not registered once + at boot. The model chooses the scope kind; it never chooses the agent or + user id, which come from the turn's resolved `Subject` and have no schema + field to override. `ScopeGlobal` is never model-writable. + +`boot.setupMemoryEngine` (`internal/app/archied/bootstrap.go`) registers and +starts the engine and must run before `setupGatewayChat`, which constructs the +only chat turn runner in production and captures the engine at construction +time (`internal/app/archied/composition_order_test.go` pins the ordering), and +before `setupCurators`, whose `Registrar` captures the engine registry. The +session curator (`internal/infrastructure/sessioncurator`) writes derived +records onto `ScopeAgentUser`, making it the engine's other live producer +besides the chat write path. + ## Plugin implementations First-party plugins are self-contained vertical modules under @@ -48,8 +139,9 @@ in-process through typed capability families (this document's model) or out-of-process like a generic `DispatchEvent` dispatcher. Decided 2026-08-22 in `docs/prds/event-sources-and-reactions.md`: in-process typed families, confirming this document's model rather than amending it. The forge webhook -receiver (`internal/forge/webhook`, see `ARCHITECTURE.md`'s "Webhook intake" -section) and the EDA playbook engine's Module action position +receiver (`internal/forge/webhook`; see "Intake provenance" in +`docs/architecture/messaging-and-work-intake.md`) and the EDA playbook engine's +Module action position (`docs/prds/eda-playbook-engine.md`, `internal/domain/eda/module`) are the shipped instances of that decision -- each is a narrow typed contract with an owning registry, not a generic event hook. The playbook engine's Channel and diff --git a/docs/data/generated/dev-docs.json b/docs/data/generated/dev-docs.json index 0a8d39231..4571c236c 100644 --- a/docs/data/generated/dev-docs.json +++ b/docs/data/generated/dev-docs.json @@ -22,7 +22,7 @@ "url": "/dev/architecture/bindings/", "title": "Playbook bindings: model and threat model", "section": "Architecture", - "body": "# Playbook bindings: model and threat model\n\n**Status:** Approved foundation (documents shipped code: `archie-core-t2db.1`\nthrough `.5`, `.7`, `.9`-`.15`)\n**Date:** 2026-09-03\n\nThis document covers the binding model and the intake threat model portion\nof `archie-core-t2db.6`. It does not cover the `ARCHITECTURE.md` task-lifecycle\naddendum or the operator walkthrough -- those remain separate, open pieces of\nthe same ticket.\n\n## What a binding is\n\nA `binding` (`internal/domain/binding.Binding`) ties four things together:\na **matcher** (which captured events it applies to), a **mapping**\n(`internal/domain/mapping.Mapping`, how fields are pulled out of a payload),\na **workflow** (which registered workflow runs on a match), and a **state\nmachine** governing whether it is live.\n\n```go\ntype Binding struct {\n ID, Name // identity\n Matcher // { Source string } -- the webhook path segment, e.g. \"sentry\"\n MappingID // which Mapping resolves payload fields\n Workflow // registered workflow name to dispatch to\n Version // bumped on every edit\n Status // draft | pending_approval | armed\n Secret // HMAC-SHA256 shared secret, encrypted at rest, never returned by GET\n}\n```\n\nBindings live in the store (SQLite), not `config.toml` -- operators author\nthem from the dashboard while the daemon runs, without a restart.\n\n### Matching\n\n`Matcher.Source` is the only predicate today: the path segment a sender\nPOSTs to (e.g. a webhook hitting `.../sentry` matches bindings with\n`Matcher.Source == \"sentry\"`). The storage column is plain `TEXT`, so adding\npredicates later is non-breaking -- extend `Matcher`, don't invent a second\nmatcher shape.\n\n### Mapping\n\n`mapping.Resolve(fields, payload)` walks a captured JSON payload against a\nlist of typed `Field`s (name, JSON path, expected type), returning resolved\nvalues plus a `[]Failure` for anything that didn't parse or type-match. This\nis schema-by-example in practice: a mapping is authored by pointing at a\nreal captured payload in the dashboard, not by guessing a schema in advance.\n\n### Versioning\n\n`Version` increments on every `UpdateBinding` call. This exists so an\neditor's later change can never silently rewrite the historical provenance\nof a task that already fired -- a dispatched task's binding reference is\npinned to the version that was armed when it fired, not \"whatever the\nbinding currently says.\"\n\n### State machine\n\n```\ndraft ---\u003e pending_approval ---\u003e armed\n ^ ^ |\n | | |\n +--------------+-----------------+\n (any edit to an armed binding drops it back to pending_approval)\n```\n\nModeled directly on Telegram's existing `dangerousAction`/`pendingApproval`\nflow (`internal/channels/telegram/dangerous.go`, `approval.go`) -- not a new\napproval mechanism invented for this feature. **Only `armed` evaluates\nagainst incoming events** (`binding.Matches`, `internal/domain/binding/\nbinding.go`); `draft` and `pending_approval` bindings are inert. An edit to\nan already-armed binding cannot bypass approval by silent mutation -- it\ndrops back to `pending_approval` and requires an explicit `Approve` call to\nre-arm.\n\n## The threat model\n\nThis is a public, unauthenticated-by-default intake surface whose output is\nan agent with a repo, a worktree, and tool access. The design brief\n(`archie-core-t2db.5`, `docs/prds/webhook-intake-security.md`) named five\nnon-negotiables; each is verified below against the actual shipped code,\nnot the design intent.\n\n### 1. Per-source authentication\n\nReuses `internal/channels/webhook`'s existing HMAC-SHA256 scheme --\ncapture accepts unsigned events (visible in the dashboard, marked\nunauthenticated), but **only authenticated events can ever match a\nbinding**. `binding.Matches`' doc comment states this plainly: \"an\nunauthenticated event can never trigger a binding, no matter how well it\nwould otherwise match.\" Enforced structurally at two points, not just\nconvention:\n\n- `captured_events.authenticated` is a real column (`internal/store/\n captures.go`), set at capture time.\n- `dispatchBindings` (`internal/daemon/daemon.go:479`) walks **authenticated\n captures whose source has at least one armed binding** -- an\n unauthenticated capture is never even considered for dispatch.\n\n### 2. Human approval before a binding goes live\n\nCovered by the state machine above. Nothing self-arms: `pending_approval -\u003e\narmed` requires an explicit `Approve` call, and any edit to an armed\nbinding reverts it.\n\n### 3. Rate and volume limits per source\n\nCapture writes are bounded independently of the approval gate, so a flood\nof unauthenticated traffic cannot exhaust disk or drown the inspector:\n`InsertCapture` (`internal/store/captures.go`) prunes by both **retention\nwindow** (`WHERE received_at \u003c cutoff`) and **max event count** in the same\ninsert transaction.\n\n### 4. Blast radius bounded by the same code-level controls every task gets\n\nA playbook-originated task is dispatched into the **same** gate, diff-cap,\nsandboxed-container, worktree-isolation pipeline every task goes through\nregardless of origin. Verified precisely, not assumed: `dispatchOneBinding`\ncalls `EnqueueBindingTask` (`internal/store/store.go`), which wraps\n`EnqueueChatTask` -- the same direct-to-`tasks`-table enqueue path\nchat-spawned tasks already use, then stamps `binding_id`/`binding_version`\nfor provenance in a second statement. This is a **different** enqueue\nmechanism from forge-issue polling's `workintake.TaskEnvelope`/NATS path\n(`docs/prds/event-sources-and-reactions.md`) -- binding and forge-issue\ntasks arrive by different producers, but both become an ordinary\n`store.Task` row that `ClaimNext` picks up identically, so both get the\nsame gate/worktree pipeline downstream regardless of which producer created\nthe row. Payload content is\nordinary untrusted prompt input; the sender's text is never treated as a\ntrusted instruction (Archie's environmental-enforcement principle: \"the\nagent should ignore instructions in the payload\" is not itself a control,\nthe surrounding sandbox is).\n\n### 5. Captured payloads may contain secrets from the sending system\n\nTwo independent protections, not one:\n\n- **Redaction before persistence**: `webhookguard.RedactPayload` walks the\n decoded JSON and redacts recognised sensitive keys before the row is\n written -- the raw, unredacted payload is never durably stored.\n- **Retention**: a configurable window (`InsertCapture`'s `retention`\n parameter) prunes old captures automatically; visibility of what remains\n is gated by the same dashboard authentication as every other webui\n surface, not a separate, weaker path.\n\n### Secrets at rest\n\n`Binding.Secret` (the HMAC shared secret a sender signs with) is encrypted\nat rest via AES-GCM (`internal/store/binding_cipher.go`, `archie-core-\nt2db.7`) -- a leaked SQLite file does not leak sender-side shared secrets in\nplaintext. The envelope is versioned (`bindingEnvelopeVersion`) so a future\ncipher or KDF change can roll forward without breaking already-encrypted\nrows, and the AAD binds ciphertext to the binding-secret context so it\ncannot be relocated to another field/column and still authenticate. A `nil`\ncipher (no encryption key configured) leaves secrets as plaintext --\ndocumented legacy behaviour, not silent degradation.\n\n### At-most-once dispatch\n\n`dispatchBindings` enqueues at most one task per `(binding, capture)` pair,\nenforced by a real ledger, not best-effort application logic:\n`RecordDispatch` (`internal/store/bindings.go`) does `INSERT OR IGNORE` into\n`binding_dispatches`, unique on `(binding_id, capture_id)`; a duplicate\ninsert returns the sentinel `ErrAlreadyDispatched`, which the dispatch loop\ntreats as \"another cycle raced us,\" not an error. This guarantee holds\n**across daemon restarts**, not just within one process's lifetime, because\nthe ledger is a durable table, not in-memory state.\n\nThis ledger is also the direct precedent for the EDA playbook engine's\nexecution-time idempotency, now landed (`archie-core-t2db.17` closed): the\n`playbook_dispatches` table in `internal/store/playbook_dispatches.go` copies\n`binding_dispatches`'s `INSERT OR IGNORE` conventions with a\n`(playbook_id, playbook_version, event_id, action_id)` key\n(`docs/prds/eda-playbook-engine.md` gap 2).\n\n## What this document does not cover\n\n- The `ARCHITECTURE.md` task-lifecycle section noting that a\n playbook-originated task has different provenance from a forge-issue\n task -- open, part of `archie-core-t2db.6`.\n- An end-to-end operator walkthrough (point an external app at archie,\n build a first playbook) -- open, part of `archie-core-t2db.6`.\n- The playbook YAML / CEL / Module/Channel/Forge action-position work\n (`docs/prds/eda-playbook-engine.md`, `module-position.md`,\n `channel-forge-positions.md`) -- a different, newer mechanism than the\n binding model described here; the two are related but not the same\n system, and this document does not attempt to unify them.\n" + "body": "# Playbook bindings: model and threat model\n\n**Status:** Approved foundation (documents shipped code: `archie-core-t2db.1`\nthrough `.5`, `.7`, `.9`-`.15`)\n**Date:** 2026-09-03\n\nThis document covers the binding model and the intake threat model portion\nof `archie-core-t2db.6`. It does not cover the task-lifecycle provenance rule\n(`messaging-and-work-intake.md`'s \"Intake provenance\") or the operator\nwalkthrough -- that walkthrough remains an open piece of the same ticket.\n\n## What a binding is\n\nA `binding` (`internal/domain/binding.Binding`) ties four things together:\na **matcher** (which captured events it applies to), a **mapping**\n(`internal/domain/mapping.Mapping`, how fields are pulled out of a payload),\na **workflow** (which registered workflow runs on a match), and a **state\nmachine** governing whether it is live.\n\n```go\ntype Binding struct {\n ID, Name // identity\n Matcher // { Source string } -- the webhook path segment, e.g. \"sentry\"\n MappingID // which Mapping resolves payload fields\n Workflow // registered workflow name to dispatch to\n Version // bumped on every edit\n Status // draft | pending_approval | armed\n Secret // HMAC-SHA256 shared secret, encrypted at rest, never returned by GET\n}\n```\n\nBindings live in the store (SQLite), not `config.toml` -- operators author\nthem from the dashboard while the daemon runs, without a restart.\n\n### Matching\n\n`Matcher.Source` is the only predicate today: the path segment a sender\nPOSTs to (e.g. a webhook hitting `.../sentry` matches bindings with\n`Matcher.Source == \"sentry\"`). The storage column is plain `TEXT`, so adding\npredicates later is non-breaking -- extend `Matcher`, don't invent a second\nmatcher shape.\n\n### Mapping\n\n`mapping.Resolve(fields, payload)` walks a captured JSON payload against a\nlist of typed `Field`s (name, JSON path, expected type), returning resolved\nvalues plus a `[]Failure` for anything that didn't parse or type-match. This\nis schema-by-example in practice: a mapping is authored by pointing at a\nreal captured payload in the dashboard, not by guessing a schema in advance.\n\n### Versioning\n\n`Version` increments on every `UpdateBinding` call. This exists so an\neditor's later change can never silently rewrite the historical provenance\nof a task that already fired -- a dispatched task's binding reference is\npinned to the version that was armed when it fired, not \"whatever the\nbinding currently says.\"\n\n### State machine\n\n```\ndraft ---\u003e pending_approval ---\u003e armed\n ^ ^ |\n | | |\n +--------------+-----------------+\n (any edit to an armed binding drops it back to pending_approval)\n```\n\nModeled directly on Telegram's existing `dangerousAction`/`pendingApproval`\nflow (`internal/channels/telegram/dangerous.go`, `approval.go`) -- not a new\napproval mechanism invented for this feature. **Only `armed` evaluates\nagainst incoming events** (`binding.Matches`, `internal/domain/binding/\nbinding.go`); `draft` and `pending_approval` bindings are inert. An edit to\nan already-armed binding cannot bypass approval by silent mutation -- it\ndrops back to `pending_approval` and requires an explicit `Approve` call to\nre-arm.\n\n## The threat model\n\nThis is a public, unauthenticated-by-default intake surface whose output is\nan agent with a repo, a worktree, and tool access. The design brief\n(`archie-core-t2db.5`, `docs/prds/webhook-intake-security.md`) named five\nnon-negotiables; each is verified below against the actual shipped code,\nnot the design intent.\n\n### 1. Per-source authentication\n\nReuses `internal/channels/webhook`'s existing HMAC-SHA256 scheme --\ncapture accepts unsigned events (visible in the dashboard, marked\nunauthenticated), but **only authenticated events can ever match a\nbinding**. `binding.Matches`' doc comment states this plainly: \"an\nunauthenticated event can never trigger a binding, no matter how well it\nwould otherwise match.\" Enforced structurally at two points, not just\nconvention:\n\n- `captured_events.authenticated` is a real column (`internal/store/\n captures.go`), set at capture time.\n- `dispatchBindings` (`internal/daemon/daemon.go:479`) walks **authenticated\n captures whose source has at least one armed binding** -- an\n unauthenticated capture is never even considered for dispatch.\n\n### 2. Human approval before a binding goes live\n\nCovered by the state machine above. Nothing self-arms: `pending_approval -\u003e\narmed` requires an explicit `Approve` call, and any edit to an armed\nbinding reverts it.\n\n### 3. Rate and volume limits per source\n\nCapture writes are bounded independently of the approval gate, so a flood\nof unauthenticated traffic cannot exhaust disk or drown the inspector:\n`InsertCapture` (`internal/store/captures.go`) prunes by both **retention\nwindow** (`WHERE received_at \u003c cutoff`) and **max event count** in the same\ninsert transaction.\n\n### 4. Blast radius bounded by the same code-level controls every task gets\n\nA playbook-originated task is dispatched into the **same** gate, diff-cap,\nsandboxed-container, worktree-isolation pipeline every task goes through\nregardless of origin. Verified precisely, not assumed: `dispatchOneBinding`\ncalls `EnqueueBindingTask` (`internal/store/store.go`), which wraps\n`EnqueueChatTask` -- the same direct-to-`tasks`-table enqueue path\nchat-spawned tasks already use, then stamps `binding_id`/`binding_version`\nfor provenance in a second statement. This is a **different** enqueue\nmechanism from forge-issue polling's `workintake.TaskEnvelope`/NATS path\n(`docs/prds/event-sources-and-reactions.md`) -- binding and forge-issue\ntasks arrive by different producers, but both become an ordinary\n`store.Task` row that `ClaimNext` picks up identically, so both get the\nsame gate/worktree pipeline downstream regardless of which producer created\nthe row. Payload content is\nordinary untrusted prompt input; the sender's text is never treated as a\ntrusted instruction (Archie's environmental-enforcement principle: \"the\nagent should ignore instructions in the payload\" is not itself a control,\nthe surrounding sandbox is).\n\n### 5. Captured payloads may contain secrets from the sending system\n\nTwo independent protections, not one:\n\n- **Redaction before persistence**: `webhookguard.RedactPayload` walks the\n decoded JSON and redacts recognised sensitive keys before the row is\n written -- the raw, unredacted payload is never durably stored.\n- **Retention**: a configurable window (`InsertCapture`'s `retention`\n parameter) prunes old captures automatically; visibility of what remains\n is gated by the same dashboard authentication as every other webui\n surface, not a separate, weaker path.\n\n### Secrets at rest\n\n`Binding.Secret` (the HMAC shared secret a sender signs with) is encrypted\nat rest via AES-GCM (`internal/store/binding_cipher.go`, `archie-core-\nt2db.7`) -- a leaked SQLite file does not leak sender-side shared secrets in\nplaintext. The envelope is versioned (`bindingEnvelopeVersion`) so a future\ncipher or KDF change can roll forward without breaking already-encrypted\nrows, and the AAD binds ciphertext to the binding-secret context so it\ncannot be relocated to another field/column and still authenticate. A `nil`\ncipher (no encryption key configured) leaves secrets as plaintext --\ndocumented legacy behaviour, not silent degradation.\n\n### At-most-once dispatch\n\n`dispatchBindings` enqueues at most one task per `(binding, capture)` pair,\nenforced by a real ledger, not best-effort application logic:\n`RecordDispatch` (`internal/store/bindings.go`) does `INSERT OR IGNORE` into\n`binding_dispatches`, unique on `(binding_id, capture_id)`; a duplicate\ninsert returns the sentinel `ErrAlreadyDispatched`, which the dispatch loop\ntreats as \"another cycle raced us,\" not an error. This guarantee holds\n**across daemon restarts**, not just within one process's lifetime, because\nthe ledger is a durable table, not in-memory state.\n\nThis ledger is also the direct precedent for the EDA playbook engine's\nexecution-time idempotency, now landed (`archie-core-t2db.17` closed): the\n`playbook_dispatches` table in `internal/store/playbook_dispatches.go` copies\n`binding_dispatches`'s `INSERT OR IGNORE` conventions with a\n`(playbook_id, playbook_version, event_id, action_id)` key\n(`docs/prds/eda-playbook-engine.md` gap 2).\n\n## What this document does not cover\n\n- An end-to-end operator walkthrough (point an external app at archie,\n build a first playbook) -- open, part of `archie-core-t2db.6`.\n- The playbook YAML / CEL / Module/Channel/Forge action-position work\n (`docs/prds/eda-playbook-engine.md`, `module-position.md`,\n `channel-forge-positions.md`) -- a different, newer mechanism than the\n binding model described here; the two are related but not the same\n system, and this document does not attempt to unify them.\n" }, { "url": "/dev/architecture/configuration/", @@ -58,7 +58,7 @@ "url": "/dev/architecture/messaging-and-work-intake/", "title": "Messaging and Work Intake", "section": "Architecture", - "body": "# Messaging and Work Intake\n\n**Status:** Current state researched; foundation decisions in progress \n**Date:** 2026-07-28 \n**Tracking issue:** [#73](https://github.com/samcharles93/archie-core/issues/73)\n\n## Purpose\n\nArchie is a personal assistant and general agentic system. Its interactions\ninclude conversation, questions, notes, reminders, goals, message handling,\ntroubleshooting, infrastructure operations, server management, source changes,\nand open-ended future capabilities.\n\nA message to an agent is therefore not inherently a work request, and invoking\nan agent is not inherently a WorkflowExecution.\n\nMessaging and work intake are separate from the Agent System's durable workflow\nexecution model:\n\n- messaging preserves interactions with people and external systems;\n- an agent turn interprets and responds to an interaction using its available\n capabilities;\n- work intake handles the optional promotion of an interaction into durable,\n workflow-backed work;\n- the Agent System owns Workflow definitions and creates WorkflowExecutions.\n\nForge issues, Jira issues, Telegram messages, emails, webhooks, and future\nchannels remain peer external message or artifact sources. None becomes an\nexecution model merely because it can lead to work.\n\n## Confirmed boundary\n\nWork intake MUST NOT create a WorkflowExecution directly.\n\nInbound interactions normally route to the Agent selected by the channel\nbinding. The selected Agent retains its own memory, user knowledge, conversation\ncontinuity, specialisation, and capability context. This continuity is\nindependent of the underlying model or provider.\n\nEach selected Agent acts through its own immutable IdentityID. Interaction and\nresulting action records preserve both that acting identity and the initiating\nuser identity when one exists.\n\nAgent memory is private by default. Interaction context MAY additionally use\nuser-wide memory and intentionally global shared memory, but another Agent's\nagent-wide memory is not part of the selected Agent's context.\n\nMultiple users may interact with the same Agent. The channel binding and inbound\nmessage MUST resolve both AgentID and initiating User IdentityID before\nconversation or memory context is loaded. Sharing an Agent MUST NOT implicitly\nshare one user's private memory with another user.\n\nAn interaction may assemble memory from four scopes: global shared, the selected\nAgent's agent-wide memory, the initiating user's authorized user-wide memory,\nand the private Agent-user relationship memory addressed by both IDs.\n\n## Conversation identity\n\nEvery conversation is identified by the complete tuple:\n\n```text\nAgentID\n+ UserIdentityID\n+ ChannelBindingID\n+ ExternalConversationID\n+ ThreadID\n```\n\n`ThreadID` has an explicit empty value for channels without threads. No element\nof this key may be replaced by a display name, configured bot username, channel\ntype, or other mutable label.\n\nAgent and user memory continue across conversations according to their memory\nscope. Immediate conversation history and delivery correlation remain isolated\nto the identified channel conversation or thread. Moving between Telegram,\nemail, or another channel does not create a new Agent or user, but it does\ncreate a distinct conversation unless an explicit linking operation relates\nthem.\n\nConversation and message events support projections for channel-entry metrics,\nuse cases, volumes, outcomes, latency, capability usage, and other product or\noperational analysis. Metrics are derived read models; they MUST NOT become\nmutable counters embedded in the conversation aggregate or change message\nrouting semantics.\n\n## Canonical messages\n\nEvery inbound, assistant, tool-call, and tool-result message has an immutable\napplication-generated MessageID. Platform identifiers are external correlation,\nnot substitutes for the canonical ID.\n\nMessages are addressable because an interaction is more than text. A message may\ncontain assistant content, reasoning data subject to its retention policy, one\nor more tool calls, a tool result with authoritative execution metadata,\nattachments, reply relationships, edits, or other typed content. Stable\nMessageIDs also allow sessions to branch from an exact prior point.\n\nThe canonical session and message implementation must supply:\n\n- application-generated stable message IDs;\n- user, assistant, and tool roles;\n- assistant tool calls and correlated tool-result messages;\n- authoritative tool-result status, timing, error, truncation, and size\n metadata;\n- session lifecycle and active-request state;\n- session-to-agent-instance and parent-session lineage;\n- durable SQLite round trips and timestamp validation.\n\nThese semantics must use the confirmed Agent, user, channel-binding, and\nconversation ownership model.\n\nThe canonical message store MUST preserve messages as immutable records and\nrepresent later edits, deletions, and branches through related records or\nevents.\n\nArchie requires exact branch-point correlation using stable MessageIDs.\n\n## Conversation branches\n\nA branch is a continuation of a previous conversation at an exact message. A\nchild conversation records:\n\n```text\nParentConversationID\nForkMessageID\n```\n\nMessages remain an append-only linear sequence within each conversation. The\nchild's effective initial context is the parent's history through\n`ForkMessageID`; messages after that point in the parent are not inherited.\n\nAfter the fork, parent and child are isolated:\n\n- child messages, tool calls, tool results, capability actions, and errors do\n not appear in the parent conversation;\n- later parent messages do not silently enter the child;\n- returning to the parent resumes its unchanged history;\n- the child may itself be branched using the same rules;\n- branch lineage remains queryable for navigation, audit, and metrics.\n\nThis supports building substantial context about a subject and then performing\none or more contextual subtasks without polluting the original conversation. For\nexample, a user may branch from a project discussion, create Jira issues in the\nchild using inherited context, and then return to the project discussion without\nadding the Jira tool transcript to the parent.\n\nBranching does not copy mutable message rows. History is resolved through the\nimmutable parent conversation and fork MessageID, followed by the child's own\nmessage sequence.\n\nBranch isolation applies to conversation history, not to Agent learning or\nmemory. A memory action performed in a branch has exactly the same semantics as\nthe same action performed in any other conversation. It writes to its selected\nglobal, agent-wide, user-wide, or Agent-user relationship scope.\n\nMemory provenance MAY retain the source ConversationID and MessageIDs for\nexplanation and audit. Retrieval and effect are determined by memory scope and\naccess rules, not by the conversation from which the memory originated.\nReturning to a parent does not insert the child's tool transcript into the\nparent history, but scoped memories learned in the child remain available to\nfuture retrieval wherever that scope is visible.\n\nChanging sessions or branching therefore controls immediate conversational\ncontext without preventing the Agent from learning durable knowledge.\n\n## Target ownership\n\nThe Messaging domain lives at `internal/domain/messaging`. It owns:\n\n- canonical Messages and typed message content;\n- Conversations, branches, and lineage;\n- channel bindings and external source correlation;\n- replies, acknowledgements, and outbound delivery attempts;\n- message and conversation lifecycle events;\n- domain-defined repository and delivery service contracts;\n- projections required for conversation navigation and metrics.\n\nWork intake is a separate cohesive domain at `internal/domain/workintake`. It\nowns admission, validation, deterministic routing, and accepted work requests\nfor the optional transition into durable workflow-backed execution.\n\nTelegram, email, webhook, forge, Jira, and future messaging systems are\ninfrastructure adapters implementing Messaging contracts. They are users of the\ndomain; they do not own Messages, Conversations, work admission, or\nWorkflowExecution semantics.\n\nThe Agent decides whether an ordinary interaction is answered directly, uses a\ncapability, changes or retrieves Agent-owned information, or proposes durable\nworkflow-backed work.\n\nExplicitly configured automation MAY enter work intake without a conversational\nAgent turn when the external trigger and admission rule already unambiguously\nrequest durable work. This avoids spending an LLM turn to rediscover a\ndeterministic routing decision.\n\nWhen an interaction requires durable workflow-backed execution:\n\n1. a channel adapter produces a channel-neutral inbound message;\n2. the acting agent handles or interprets the interaction;\n3. work intake validates, admits, and routes a proposed work request;\n4. work intake emits an accepted work request with its source correlation,\n acting identity, requested outcome, inputs, and routing evidence;\n5. the Agent System consumes that request, selects a specific Workflow version,\n and creates the WorkflowExecution.\n\nThe accepted work request is a cross-domain handoff, not an alternate execution\naggregate. The Agent System is the sole authority that creates a valid\nWorkflowExecution.\n\nAn interaction MAY instead be completed directly by an agent turn, delegated to\na tool or capability, recorded as a note or reminder, answered conversationally,\nor retained for later context. Those paths MUST NOT require a Workflow or\nWorkflowExecution.\n\nA Workflow MAY add reusable, versioned structure to any of those behaviours.\nScheduled personal-assistant routines, message triage, reminders, infrastructure\noperations, and other capability sequences are valid Workflow definitions. The\ndistinction is whether durable structured execution is useful, not whether the\nsubject originated from a forge.\n\n## Canonical execution language\n\n- **Workflow**: reusable, versioned behaviour definition.\n- **WorkflowExecution**: durable execution of a specific Workflow version.\n- **WorkflowStep**: a defined operation in a Workflow.\n- **StepExecution**: the recorded execution and outcome of a WorkflowStep.\n- **Attempt**: a retry within the same WorkflowExecution.\n\nExternal issues, messages, requests, goals, and notes are not synonyms for these\nconcepts.\n\n## Current implementation\n\nCurrent intake is fragmented:\n\n- Telegram, email, and webhook adapters translate payloads into\n `gateway.Message`, while an unused richer `MessageEvent` contract exists\n alongside it.\n- `gateway.Router` combines local channel commands, conversational LLM routing,\n and direct task lifecycle operations.\n- Telegram is the only channel with production LLM and durable conversation\n wiring. Email and webhook free text currently have no LLM responder.\n- Forge polling bypasses the gateway and directly enqueues store rows or NATS\n task messages.\n- Chat `/spawn` directly creates a `store.Task` and invents a synthetic forge\n issue number to satisfy the existing persistence key.\n- Forge replies and chat approval commands implement separate waiting-human\n paths.\n- Message delivery, acknowledgement, retry, deduplication, and source\n correlation are platform-specific side effects rather than one durable model.\n- `store.Task` collapses an external source reference and durable execution\n state.\n\n## Current hazards\n\n- Treating every agent interaction as a work request would remove Archie's\n personal-assistant role and force conversation and immediate capability use\n through an inappropriate workflow lifecycle.\n- Treating every accepted work request as a WorkflowExecution would let intake\n or channel code select versions and construct invalid execution state outside\n the Agent System.\n- Source identity is incomplete: sessions use configured bot usernames, tasks\n use identity strings, and external message IDs are generally absent.\n- SQLite and NATS deduplication are based on forge coordinates rather than a\n channel-neutral source-message reference.\n- Replies and acknowledgements have no durable delivery-attempt lifecycle, so\n partial success and retry behaviour differ by adapter.\n- Production code has two competing channel-neutral message contracts.\n- The existing Archie session implementation lacks the required tool-call\n semantics and exact branch lineage and MUST be brought to the canonical\n contract.\n\n## Decisions still required\n\n- the cohesive owner and final location of messaging behaviour;\n- the cohesive owner and final location of work-intake admission and routing;\n- the canonical inbound message, conversation, and source-correlation model;\n- the boundary between an agent turn, direct capability use, and a proposed\n durable work request;\n- the exact relationship between an Agent, its acting IdentityID, its user, and\n its channel bindings;\n- outbound response, acknowledgement, delivery-attempt, retry, deduplication,\n and failure semantics;\n- the smallest accepted-work-request contract consumed by the Agent System;\n- migration of forge polling, replies, chat commands, email, webhook, Telegram,\n and Jira integrations.\n\n## Non-goal: semantic search over chat messages\n\nSemantic/vector search over chat messages is an **explicit non-goal** until usage\ndata shows a real need.\n\nMessage search is lexical only: session-scoped SQLite FTS5 queries paged via\n`MessageQuery{Query,Limit,Offset}` → `MessagePage`. Do not add embeddings\nconfiguration or vector columns to message records.\n\nThe rationale is cost without a demonstrated use case. Measured 2026-08-03 with\n`embeddinggemma:300m` — note this was never an infrastructure problem, the model\nwas already pulled and `ai-sdk/provider/ollama` already has `embed.go`:\n\n- 121 ms median warm per message on the write path, 12.5 s cold;\n- 768 dims = 3072 bytes per vector, roughly 15× a typical message payload;\n- chronological reads would carry vector data they do not use on the hot path\n of every turn;\n- similarity search requires an explicit indexing and retention design.\n\nIf this is revisited, vectors belong in a sibling table so chronological reads\nnever load them.\n" + "body": "# Messaging and Work Intake\n\n**Status:** Current state researched; foundation decisions in progress \n**Date:** 2026-07-28 \n**Tracking issue:** [#73](https://github.com/samcharles93/archie-core/issues/73)\n\n## Purpose\n\nArchie is a personal assistant and general agentic system. Its interactions\ninclude conversation, questions, notes, reminders, goals, message handling,\ntroubleshooting, infrastructure operations, server management, source changes,\nand open-ended future capabilities.\n\nA message to an agent is therefore not inherently a work request, and invoking\nan agent is not inherently a WorkflowExecution.\n\nMessaging and work intake are separate from the Agent System's durable workflow\nexecution model:\n\n- messaging preserves interactions with people and external systems;\n- an agent turn interprets and responds to an interaction using its available\n capabilities;\n- work intake handles the optional promotion of an interaction into durable,\n workflow-backed work;\n- the Agent System owns Workflow definitions and creates WorkflowExecutions.\n\nForge issues, Jira issues, Telegram messages, emails, webhooks, and future\nchannels remain peer external message or artifact sources. None becomes an\nexecution model merely because it can lead to work.\n\n## Confirmed boundary\n\nWork intake MUST NOT create a WorkflowExecution directly.\n\nInbound interactions normally route to the Agent selected by the channel\nbinding. The selected Agent retains its own memory, user knowledge, conversation\ncontinuity, specialisation, and capability context. This continuity is\nindependent of the underlying model or provider.\n\nEach selected Agent acts through its own immutable IdentityID. Interaction and\nresulting action records preserve both that acting identity and the initiating\nuser identity when one exists.\n\nAgent memory is private by default. Interaction context MAY additionally use\nuser-wide memory and intentionally global shared memory, but another Agent's\nagent-wide memory is not part of the selected Agent's context.\n\nMultiple users may interact with the same Agent. The channel binding and inbound\nmessage MUST resolve both AgentID and initiating User IdentityID before\nconversation or memory context is loaded. Sharing an Agent MUST NOT implicitly\nshare one user's private memory with another user.\n\nAn interaction may assemble memory from four scopes: global shared, the selected\nAgent's agent-wide memory, the initiating user's authorized user-wide memory,\nand the private Agent-user relationship memory addressed by both IDs.\n\n## Conversation identity\n\nEvery conversation is identified by the complete tuple:\n\n```text\nAgentID\n+ UserIdentityID\n+ ChannelBindingID\n+ ExternalConversationID\n+ ThreadID\n```\n\n`ThreadID` has an explicit empty value for channels without threads. No element\nof this key may be replaced by a display name, configured bot username, channel\ntype, or other mutable label.\n\nAgent and user memory continue across conversations according to their memory\nscope. Immediate conversation history and delivery correlation remain isolated\nto the identified channel conversation or thread. Moving between Telegram,\nemail, or another channel does not create a new Agent or user, but it does\ncreate a distinct conversation unless an explicit linking operation relates\nthem.\n\nConversation and message events support projections for channel-entry metrics,\nuse cases, volumes, outcomes, latency, capability usage, and other product or\noperational analysis. Metrics are derived read models; they MUST NOT become\nmutable counters embedded in the conversation aggregate or change message\nrouting semantics.\n\n## Canonical messages\n\nEvery inbound, assistant, tool-call, and tool-result message has an immutable\napplication-generated MessageID. Platform identifiers are external correlation,\nnot substitutes for the canonical ID.\n\nMessages are addressable because an interaction is more than text. A message may\ncontain assistant content, reasoning data subject to its retention policy, one\nor more tool calls, a tool result with authoritative execution metadata,\nattachments, reply relationships, edits, or other typed content. Stable\nMessageIDs also allow sessions to branch from an exact prior point.\n\nThe canonical session and message implementation must supply:\n\n- application-generated stable message IDs;\n- user, assistant, and tool roles;\n- assistant tool calls and correlated tool-result messages;\n- authoritative tool-result status, timing, error, truncation, and size\n metadata;\n- session lifecycle and active-request state;\n- session-to-agent-instance and parent-session lineage;\n- durable SQLite round trips and timestamp validation.\n\nThese semantics must use the confirmed Agent, user, channel-binding, and\nconversation ownership model.\n\nThe canonical message store MUST preserve messages as immutable records and\nrepresent later edits, deletions, and branches through related records or\nevents.\n\nArchie requires exact branch-point correlation using stable MessageIDs.\n\n## Conversation branches\n\nA branch is a continuation of a previous conversation at an exact message. A\nchild conversation records:\n\n```text\nParentConversationID\nForkMessageID\n```\n\nMessages remain an append-only linear sequence within each conversation. The\nchild's effective initial context is the parent's history through\n`ForkMessageID`; messages after that point in the parent are not inherited.\n\nAfter the fork, parent and child are isolated:\n\n- child messages, tool calls, tool results, capability actions, and errors do\n not appear in the parent conversation;\n- later parent messages do not silently enter the child;\n- returning to the parent resumes its unchanged history;\n- the child may itself be branched using the same rules;\n- branch lineage remains queryable for navigation, audit, and metrics.\n\nThis supports building substantial context about a subject and then performing\none or more contextual subtasks without polluting the original conversation. For\nexample, a user may branch from a project discussion, create Jira issues in the\nchild using inherited context, and then return to the project discussion without\nadding the Jira tool transcript to the parent.\n\nBranching does not copy mutable message rows. History is resolved through the\nimmutable parent conversation and fork MessageID, followed by the child's own\nmessage sequence.\n\nBranch isolation applies to conversation history, not to Agent learning or\nmemory. A memory action performed in a branch has exactly the same semantics as\nthe same action performed in any other conversation. It writes to its selected\nglobal, agent-wide, user-wide, or Agent-user relationship scope.\n\nMemory provenance MAY retain the source ConversationID and MessageIDs for\nexplanation and audit. Retrieval and effect are determined by memory scope and\naccess rules, not by the conversation from which the memory originated.\nReturning to a parent does not insert the child's tool transcript into the\nparent history, but scoped memories learned in the child remain available to\nfuture retrieval wherever that scope is visible.\n\nChanging sessions or branching therefore controls immediate conversational\ncontext without preventing the Agent from learning durable knowledge.\n\n## Target ownership\n\nThe Messaging domain lives at `internal/domain/messaging`. It owns:\n\n- canonical Messages and typed message content;\n- Conversations, branches, and lineage;\n- channel bindings and external source correlation;\n- replies, acknowledgements, and outbound delivery attempts;\n- message and conversation lifecycle events;\n- domain-defined repository and delivery service contracts;\n- projections required for conversation navigation and metrics.\n\nWork intake is a separate cohesive domain at `internal/domain/workintake`. It\nowns admission, validation, deterministic routing, and accepted work requests\nfor the optional transition into durable workflow-backed execution.\n\nTelegram, email, webhook, forge, Jira, and future messaging systems are\ninfrastructure adapters implementing Messaging contracts. They are users of the\ndomain; they do not own Messages, Conversations, work admission, or\nWorkflowExecution semantics.\n\nThe Agent decides whether an ordinary interaction is answered directly, uses a\ncapability, changes or retrieves Agent-owned information, or proposes durable\nworkflow-backed work.\n\nExplicitly configured automation MAY enter work intake without a conversational\nAgent turn when the external trigger and admission rule already unambiguously\nrequest durable work. This avoids spending an LLM turn to rediscover a\ndeterministic routing decision.\n\nWhen an interaction requires durable workflow-backed execution:\n\n1. a channel adapter produces a channel-neutral inbound message;\n2. the acting agent handles or interprets the interaction;\n3. work intake validates, admits, and routes a proposed work request;\n4. work intake emits an accepted work request with its source correlation,\n acting identity, requested outcome, inputs, and routing evidence;\n5. the Agent System consumes that request, selects a specific Workflow version,\n and creates the WorkflowExecution.\n\nThe accepted work request is a cross-domain handoff, not an alternate execution\naggregate. The Agent System is the sole authority that creates a valid\nWorkflowExecution.\n\nAn interaction MAY instead be completed directly by an agent turn, delegated to\na tool or capability, recorded as a note or reminder, answered conversationally,\nor retained for later context. Those paths MUST NOT require a Workflow or\nWorkflowExecution.\n\nA Workflow MAY add reusable, versioned structure to any of those behaviours.\nScheduled personal-assistant routines, message triage, reminders, infrastructure\noperations, and other capability sequences are valid Workflow definitions. The\ndistinction is whether durable structured execution is useful, not whether the\nsubject originated from a forge.\n\n## Canonical execution language\n\n- **Workflow**: reusable, versioned behaviour definition.\n- **WorkflowExecution**: durable execution of a specific Workflow version.\n- **WorkflowStep**: a defined operation in a Workflow.\n- **StepExecution**: the recorded execution and outcome of a WorkflowStep.\n- **Attempt**: a retry within the same WorkflowExecution.\n\nExternal issues, messages, requests, goals, and notes are not synonyms for these\nconcepts.\n\n## Current implementation\n\nCurrent intake is fragmented:\n\n- Telegram, email, and webhook adapters translate payloads into\n `gateway.Message`, while an unused richer `MessageEvent` contract exists\n alongside it.\n- `gateway.Router` combines local channel commands, conversational LLM routing,\n and direct task lifecycle operations.\n- Telegram is the only channel with production LLM and durable conversation\n wiring. Email and webhook free text currently have no LLM responder.\n- Forge polling bypasses the gateway and directly enqueues store rows or NATS\n task messages.\n- Chat `/spawn` directly creates a `store.Task` and invents a synthetic forge\n issue number to satisfy the existing persistence key.\n- Forge replies and chat approval commands implement separate waiting-human\n paths.\n- Message delivery, acknowledgement, retry, deduplication, and source\n correlation are platform-specific side effects rather than one durable model.\n- `store.Task` collapses an external source reference and durable execution\n state.\n\n## Intake provenance\n\nA task's origin determines which enqueue path created it, though every origin\nconverges on the same `tasks` table and the same `Route`/`Run` lifecycle once\nqueued.\n\n- Forge-sourced work -- discovered by polling or, in single-identity\n deployments, delivered by the forge webhook receiver (`internal/forge/webhook`)\n -- enqueues through the same `workintake.TaskEnvelope` and `PublishTask` path.\n `TaskEnvelope.IdempotencyKey` is keyed on `owner/repo/number` and not on the\n delivery source, so a webhook-delivered issue and a later poll of the same\n issue dedup through `PublishUnique` instead of double-enqueuing.\n- Chat-spawned and playbook-binding-dispatched work enqueues through\n `EnqueueChatTask`/`EnqueueBindingTask` (`internal/store`), stamped with\n `binding_id`/`binding_version` for provenance.\n\nWebhook intake is opt-in per deployment (`Forge.Intake` = `poll` (default),\n`webhook`, or `both`) and single-identity only: a config naming\n`webhook`/`both` alongside `[[identities]]`, or setting intake on a\n`[identities.forge]` entry, is rejected at validation rather than started with\nthe setting ignored. One receiver binds one secret and one dispatch predicate,\nso it cannot judge a second identity's events by the right rules. A\nmulti-identity deployment polls per identity until per-identity intake lands.\nThe decided contract is `docs/prds/event-sources-and-reactions.md`; the binding\nmodel and its threat model are `docs/architecture/bindings.md`.\n\n## Current hazards\n\n- Treating every agent interaction as a work request would remove Archie's\n personal-assistant role and force conversation and immediate capability use\n through an inappropriate workflow lifecycle.\n- Treating every accepted work request as a WorkflowExecution would let intake\n or channel code select versions and construct invalid execution state outside\n the Agent System.\n- Source identity is incomplete: sessions use configured bot usernames, tasks\n use identity strings, and external message IDs are generally absent.\n- SQLite and NATS deduplication are based on forge coordinates rather than a\n channel-neutral source-message reference.\n- Replies and acknowledgements have no durable delivery-attempt lifecycle, so\n partial success and retry behaviour differ by adapter.\n- Production code has two competing channel-neutral message contracts.\n- The existing Archie session implementation lacks the required tool-call\n semantics and exact branch lineage and MUST be brought to the canonical\n contract.\n\n## Decisions still required\n\n- the cohesive owner and final location of messaging behaviour;\n- the cohesive owner and final location of work-intake admission and routing;\n- the canonical inbound message, conversation, and source-correlation model;\n- the boundary between an agent turn, direct capability use, and a proposed\n durable work request;\n- the exact relationship between an Agent, its acting IdentityID, its user, and\n its channel bindings;\n- outbound response, acknowledgement, delivery-attempt, retry, deduplication,\n and failure semantics;\n- the smallest accepted-work-request contract consumed by the Agent System;\n- migration of forge polling, replies, chat commands, email, webhook, Telegram,\n and Jira integrations.\n\n## Non-goal: semantic search over chat messages\n\nSemantic/vector search over chat messages is an **explicit non-goal** until usage\ndata shows a real need.\n\nMessage search is lexical only: session-scoped SQLite FTS5 queries paged via\n`MessageQuery{Query,Limit,Offset}` → `MessagePage`. Do not add embeddings\nconfiguration or vector columns to message records.\n\nThe rationale is cost without a demonstrated use case. Measured 2026-08-03 with\n`embeddinggemma:300m` — note this was never an infrastructure problem, the model\nwas already pulled and `ai-sdk/provider/ollama` already has `embed.go`:\n\n- 121 ms median warm per message on the write path, 12.5 s cold;\n- 768 dims = 3072 bytes per vector, roughly 15× a typical message payload;\n- chronological reads would carry vector data they do not use on the hot path\n of every turn;\n- similarity search requires an explicit indexing and retention design.\n\nIf this is revisited, vectors belong in a sibling table so chronological reads\nnever load them.\n" }, { "url": "/dev/architecture/migration-decisions/", @@ -88,7 +88,7 @@ "url": "/dev/architecture/plugins-and-extensions/", "title": "Plugins and Extensions", "section": "Architecture", - "body": "# Plugins and Extensions\n\n**Status:** Approved foundation \n**Date:** 2026-07-28 \n**Tracking issue:** [#73](https://github.com/samcharles93/archie-core/issues/73)\n\n## Plugin framework\n\nThe core plugin domain lives under `internal/domain/plugin`. It owns:\n\n- plugin identity and metadata;\n- discovery and registration contracts;\n- lifecycle rules;\n- capability registration;\n- validation and compatibility rules.\n\nThe generic plugin contract remains metadata-only. Functionality is exposed\nthrough narrow typed capability contracts with an owning registry or manager.\n\nWorkflow plugins belong to the Workflow capability family. The Workflow domain\nat `internal/domain/workflow` owns their typed registration, validation,\nversioning, definition, and execution contracts. The generic plugin domain may\nprovide plugin identity, discovery, compatibility, and lifecycle mechanics, but\nit MUST NOT define Workflow semantics or expose untyped Workflow hooks.\n\n## Plugin implementations\n\nFirst-party plugins are self-contained vertical modules under\n`plugins/\u003cplugin\u003e`.\n\nA plugin:\n\n- owns its implementation, settings, templates, assets, and tests;\n- receives only explicitly supplied contracts and services;\n- MUST NOT receive daemon internals, a service locator, or unrestricted\n application access;\n- MUST NOT import arbitrary infrastructure or application composition code;\n- MAY use approved domain and shared application contracts;\n- MUST remain removable without changes to unrelated domains.\n\nPlugins may be internally cohesive like small services. They do not become\nnetwork services without a concrete security or operational reason.\n\n## Event sources and reactions\n\n`archie-core-7d5u.1` asked whether extensions should react to events\nin-process through typed capability families (this document's model) or\nout-of-process like a generic `DispatchEvent` dispatcher. Decided 2026-08-22\nin `docs/prds/event-sources-and-reactions.md`: in-process typed families,\nconfirming this document's model rather than amending it. The forge webhook\nreceiver (`internal/forge/webhook`, see `ARCHITECTURE.md`'s \"Webhook intake\"\nsection) and the EDA playbook engine's Module action position\n(`docs/prds/eda-playbook-engine.md`, `internal/domain/eda/module`) are the\nshipped instances of that decision -- each is a narrow typed contract with an\nowning registry, not a generic event hook. The playbook engine's Channel and\nForge action positions are designed but not yet implemented (open\ninvestigation `archie-core-t2db.19`); when they land they follow the same\nmodel, not a new one.\n\n## Curator surprisal sampling\n\n**Decided 2026-08-05, wave 1 landed 2026-09-05** — the curator engine family\n(epic [#435](https://github.com/samcharles93/archie-core/issues/435)) selects which memories deserve agentic attention via\nsurprisal-based sampling: score memories by how surprising they are and\nspend the sampled reasoner's agentic budget on the most surprising items.\n\nThe prerequisites are built: the embeddings capability\n([#436](https://github.com/samcharles93/archie-core/issues/436), `internal/domain/embedding` contract +\n`internal/infrastructure/embedding` implementation, config-driven via\n`models.embedding`) and the wave-1 `Sampler` strategy seam\n(`internal/domain/sampling` -- not `internal/curator`, see\n`docs/prds/curator-sampler-wave1.md` for the settled design and tracking)\nwith four\ncheap, embedding-free strategies (recency, random, all, staleness proxy).\nNo curator consumes a `Sampler` yet, and the embeddings client is likewise\ndeliberately inert: `setupEmbeddings` constructs it at boot and holds it on\n`b.embeddings`, but no production code reads that field yet, so a resolved\n`models.embedding` role currently produces a client whose only observable\neffect is a boot log line. Both are prerequisites waiting on #437, not\nunwired defects.\n\nStill deferred: [#437](https://github.com/samcharles93/archie-core/issues/437), the embedding-backed surprisal strategy, which\ndepends on the above and on [#407](https://github.com/samcharles93/archie-core/issues/407). Requirements for that deferred work,\nso it can be picked up without re-deriving the design:\n\n- A `Sampler` implementation behind the same interface as the cheap\n strategies; selection deterministic given fixed inputs.\n- Algorithm: sample up to a candidate cap; build a k-NN graph over embedding\n vectors; row-normalize the adjacency with self-loops; power-iterate to the\n stationary distribution; surprisal = −log(stationary probability of the\n nearest point); select the highest-surprisal items.\n- Degradation: content without an embedding is skipped; an embedding failure\n aborts sampling and falls back to free exploration — a sampling failure\n must never fail or crash a curator pass.\n- Cost bound: the graph build is quadratic in the candidate cap; the cap must\n be a named constant with a documented bound.\n" + "body": "# Plugins and Extensions\n\n**Status:** Approved foundation \n**Date:** 2026-07-28 \n**Tracking issue:** [#73](https://github.com/samcharles93/archie-core/issues/73)\n\n## Plugin framework\n\nThe core plugin domain lives under `internal/domain/plugin`. It owns:\n\n- plugin identity and metadata;\n- discovery and registration contracts;\n- lifecycle rules;\n- capability registration;\n- validation and compatibility rules.\n\nThe generic plugin contract remains metadata-only. Functionality is exposed\nthrough narrow typed capability contracts with an owning registry or manager.\n\nWorkflow plugins belong to the Workflow capability family. The Workflow domain\nat `internal/domain/workflow` owns their typed registration, validation,\nversioning, definition, and execution contracts. The generic plugin domain may\nprovide plugin identity, discovery, compatibility, and lifecycle mechanics, but\nit MUST NOT define Workflow semantics or expose untyped Workflow hooks.\n\n## Plugin engine rule (strict)\n\n“Engine” is reserved for a typed, lifecycle-managed capability family. It is not\na synonym for a generic plugin hook. Every new plugin engine must satisfy all of\nthese invariants:\n\n1. **Typed domain contract.** The engine interface exposes real\n capability-specific operations in addition to identity metadata. Examples are\n resolving a secret, delivering a message, or running an agent. A plugin that\n only exposes `Name` and `Version` is metadata, not an engine.\n2. **Owning family manager.** The capability package owns a `Registry` or\n `Manager` that controls provider registration and discovery, initialization,\n or access. Providers do not register arbitrary callbacks directly on the\n daemon.\n3. **Lifecycle and health.** When an implementation owns resources or background\n work, its typed family contract must expose start, health, and stop\n semantics. The owning manager invokes them and defines failure isolation and\n shutdown ordering. Stateless providers may make these operations explicit\n no-ops.\n4. **Narrow host access.** Plugins receive the typed family registrar and only\n the host services their declared capability requires. They never receive\n `*daemon.Daemon`, a service locator, or an untyped hook map.\n5. **Metadata-only generic plugin contract.** `plugin.Plugin` remains limited to\n `Name()` and `Version()`. Domain hooks belong on typed engine interfaces;\n adding capability methods to the generic interface is forbidden.\n6. **Trust boundary stays explicit.** Explicitly trusted, operator-installed\n plugins—including third-party secret-engine plugins the operator chooses to\n trust—may run in-process and have daemon privileges. Repository-supplied code\n runs in the task container. Integrations that are not trusted as daemon code\n run out of process through MCP or a versioned protocol and, when a\n confidentiality or integrity boundary is required, inside a real container\n sandbox. A subprocess boundary alone is not a security sandbox, and calling\n Yaegi-loaded code a plugin does not isolate it.\n\n`internal/plugin/architecture_test.go` enforces the mechanically testable parts\nof this rule: the generic plugin method set, the shared agent-instruction\nsource, and the requirement that engine interfaces have domain behavior plus an\nowning registry or manager.\n\n### Memory engine family\n\nOne engine, addressed by four typed scopes -- `ScopeGlobal`, `ScopeAgent`,\n`ScopeUser`, `ScopeAgentUser` (`internal/domain/memory`, `Scope`/`ScopeKind`).\nA caller names exactly the scopes it may read or write; the engine applies no\naccess policy of its own, so isolation falls out mechanically because\n`ScopeAgent{a}` and `ScopeAgent{b}` are different storage keys. See\n`docs/prds/memory-engine-unification.md` for the decision this contract\nimplements; it superseded `internal/memory` (`MemoryProvider`/`Manager`),\ndeleted in that PRD's slice 5.\n\n`MemoryEngine` (`internal/domain/memory/contract.go`) is the typed family\ncontract: `Create`, `Get`, `Query`, `List`, `Update`, `Forget`, `Revisions` --\nCRUD with revisions retained, not just create/read/delete. `Update` supersedes\nrather than overwrites: the superseded state is appended to the scope's\n`HISTORY.md` before the live block is replaced, and `Update` carries no\n`Scope` field, so a record can never change scope. The family's content\nscanner (prompt-injection / sensitive-data patterns) is applied inside\n`Create` and `Update`, the single choke point every producer crosses.\n\n`internal/domain/memory.Registry` is the owning family registry: registration,\nstart/health/stop, shutdown ordering, and failure isolation, following the\nplugin engine rule above. `internal/infrastructure/memory.NewBuiltinEngine` is\nthe only registered engine today; it persists through\n`internal/infrastructure/memory/builtin`, a markdown store (`MEMORY.md`-shaped\nper-scope files) that owns its on-disk format end-to-end.\n\nTwo per-turn seams in `internal/gateway` connect a chat turn to the engine,\neach narrowed to exactly the operations it needs:\n\n- **Read** (`turn_memory.go`, `MemoryStore`): `prepareTurn` runs a\n scope-only `Query` over the resolved `Subject`'s readable scopes,\n synchronously and before the prompt is built. It never returns an error --\n a panicking or failing engine degrades to an empty `\u003cmemory\u003e` block rather\n than failing the turn -- and is bounded by a per-scope record limit and a\n byte cap on the rendered block.\n- **Write** (`turn_memory_tool.go`, `MemoryWriteStore`): four tools --\n `memory_create`, `memory_update`, `memory_delete`, `memory_list` -- built\n per turn onto the resolved `Subject`'s writable scopes, not registered once\n at boot. The model chooses the scope kind; it never chooses the agent or\n user id, which come from the turn's resolved `Subject` and have no schema\n field to override. `ScopeGlobal` is never model-writable.\n\n`boot.setupMemoryEngine` (`internal/app/archied/bootstrap.go`) registers and\nstarts the engine and must run before `setupGatewayChat`, which constructs the\nonly chat turn runner in production and captures the engine at construction\ntime (`internal/app/archied/composition_order_test.go` pins the ordering), and\nbefore `setupCurators`, whose `Registrar` captures the engine registry. The\nsession curator (`internal/infrastructure/sessioncurator`) writes derived\nrecords onto `ScopeAgentUser`, making it the engine's other live producer\nbesides the chat write path.\n\n## Plugin implementations\n\nFirst-party plugins are self-contained vertical modules under\n`plugins/\u003cplugin\u003e`.\n\nA plugin:\n\n- owns its implementation, settings, templates, assets, and tests;\n- receives only explicitly supplied contracts and services;\n- MUST NOT receive daemon internals, a service locator, or unrestricted\n application access;\n- MUST NOT import arbitrary infrastructure or application composition code;\n- MAY use approved domain and shared application contracts;\n- MUST remain removable without changes to unrelated domains.\n\nPlugins may be internally cohesive like small services. They do not become\nnetwork services without a concrete security or operational reason.\n\n## Event sources and reactions\n\n`archie-core-7d5u.1` asked whether extensions should react to events\nin-process through typed capability families (this document's model) or\nout-of-process like a generic `DispatchEvent` dispatcher. Decided 2026-08-22\nin `docs/prds/event-sources-and-reactions.md`: in-process typed families,\nconfirming this document's model rather than amending it. The forge webhook\nreceiver (`internal/forge/webhook`; see \"Intake provenance\" in\n`docs/architecture/messaging-and-work-intake.md`) and the EDA playbook engine's\nModule action position\n(`docs/prds/eda-playbook-engine.md`, `internal/domain/eda/module`) are the\nshipped instances of that decision -- each is a narrow typed contract with an\nowning registry, not a generic event hook. The playbook engine's Channel and\nForge action positions are designed but not yet implemented (open\ninvestigation `archie-core-t2db.19`); when they land they follow the same\nmodel, not a new one.\n\n## Curator surprisal sampling\n\n**Decided 2026-08-05, wave 1 landed 2026-09-05** — the curator engine family\n(epic [#435](https://github.com/samcharles93/archie-core/issues/435)) selects which memories deserve agentic attention via\nsurprisal-based sampling: score memories by how surprising they are and\nspend the sampled reasoner's agentic budget on the most surprising items.\n\nThe prerequisites are built: the embeddings capability\n([#436](https://github.com/samcharles93/archie-core/issues/436), `internal/domain/embedding` contract +\n`internal/infrastructure/embedding` implementation, config-driven via\n`models.embedding`) and the wave-1 `Sampler` strategy seam\n(`internal/domain/sampling` -- not `internal/curator`, see\n`docs/prds/curator-sampler-wave1.md` for the settled design and tracking)\nwith four\ncheap, embedding-free strategies (recency, random, all, staleness proxy).\nNo curator consumes a `Sampler` yet, and the embeddings client is likewise\ndeliberately inert: `setupEmbeddings` constructs it at boot and holds it on\n`b.embeddings`, but no production code reads that field yet, so a resolved\n`models.embedding` role currently produces a client whose only observable\neffect is a boot log line. Both are prerequisites waiting on #437, not\nunwired defects.\n\nStill deferred: [#437](https://github.com/samcharles93/archie-core/issues/437), the embedding-backed surprisal strategy, which\ndepends on the above and on [#407](https://github.com/samcharles93/archie-core/issues/407). Requirements for that deferred work,\nso it can be picked up without re-deriving the design:\n\n- A `Sampler` implementation behind the same interface as the cheap\n strategies; selection deterministic given fixed inputs.\n- Algorithm: sample up to a candidate cap; build a k-NN graph over embedding\n vectors; row-normalize the adjacency with self-loops; power-iterate to the\n stationary distribution; surprisal = −log(stationary probability of the\n nearest point); select the highest-surprisal items.\n- Degradation: content without an embedding is skipped; an embedding failure\n aborts sampling and falls back to free exploration — a sampling failure\n must never fail or crash a curator pass.\n- Cost bound: the graph build is quadratic in the candidate cap; the cap must\n be a named constant with a documented bound.\n" }, { "url": "/dev/architecture/policy/", @@ -202,7 +202,7 @@ "url": "/dev/prds/deep-research-capability/", "title": "Deep research capability — design", "section": "PRDs", - "body": "# Deep research capability — design\n\n**Status:** Draft\n**Date:** 2026-09-14\n**Scope:** A deep-research capability in archie-core: an orchestrator that\nspawns 1..N subagents to explore and research in parallel, collates their\nfindings, and returns one cited report. Grounded in web search and web fetch.\n**Beads issue:** none yet (this document is the input to it)\n**Depends on:** nothing shipped for slice 1; slice 2 depends on slice 1.\n\n## §0 Implementation status\n\nEvery claim in this document is mapped to what exists in the tree.\n\n| Claim | Status | Where it stands |\n|---|---|---|\n| `web_fetch` tool (URL → readable text) | **Implemented** | `internal/tools/webfetch/{tool.go,webfetch.go}`; registered in `internal/app/archied/bootstrap.go` `registerStandaloneTools`, disabled-by-config returns `nil` |\n| **Web search** (query → result list) | **Aspirational — zero code** | No search provider, no search tool, no config block anywhere in the tree or in `ai-sdk` v0.1.32 |\n| `agent.Subagent` — nested agent in a fresh context window | **Implemented** (dependency) | `github.com/samcharles93/ai-sdk@v0.1.32/agent/subagent.go`. Synchronous, non-streaming, own model/system/toolset, `MaxSteps` defaults to 10, nesting depth capped at 5, returns final text only |\n| Subagent used in production | **Implemented, one consumer** | `internal/app/agentworker/review.go` — the adversarial reviewer. That is the only non-test use of `Subagent` in the repo |\n| A **delegate/subagent tool** on the chat surface | **Aspirational — zero code** | No `subagent`/`delegate` tool name is registered anywhere |\n| **Fan-out orchestration** (N missions, one collator) | **Aspirational — zero code** | `agentloop`'s package doc states it runs *one* mission: \"orchestration across missions lives with the caller\". No caller does this yet |\n| Parallel-safe tool execution | **Aspirational — zero code** | The unreachable `internal/tools/dispatch.go` dispatch/gating engine (`ExecutionClass`, `DispatchSequential`/`DispatchConcurrent`/`ClassifyEntry`) was deleted as dead code (tools-core-1); live concurrency is ai-sdk's `GenerateOptions.MaxParallelToolCalls`, which archie-core never sets |\n| Per-result caps + spill-to-disk | **Implemented** | `internal/agentexec/toolset.go` `ToolLimits{MaxResultChars,SpillDir}`, applied in `toolExecute` via `tools.CapPayload` (`limit.go`); no **aggregate** turn cap by design (`toolset.go`). `config.ToolPolicy.SpillDir`; default cap 50k chars, spill dir deliberately not defaulted |\n| Tool guardrails (repeat-failure / no-progress) | **Implemented, not on the call path** | `internal/tools/guardrail.go` engine exists. Its only non-test caller is `dispatchOne` — which is itself unreachable — so **the chat path never consults it**, and the agent path only records per *stage* (`domain/workflow/agent.go`) |\n| Structured capture tools (model reports into a typed store, not prose) | **Implemented** | `internal/app/agentworker/review.go` `record_finding` / `record_checked` — `core.NewTypedTool` appending to a caller-owned slice. This is the pattern a research source ledger should reuse |\n| Curator engine (interval, declared tool set, lifecycle) | **Implemented, tools broken** | `internal/domain/curator`; a curator declaring tools is **refused** at registration (`registry.go`, no `ToolBuilder` implementation) and `curatorLLMRunner` (`internal/app/archied/main.go`) drops `ChatRequest.Tools`. Budgets are time+concurrency only (`runtime.go`) — **no token budget** |\n| Cron / scheduling + delivery router | **Implemented, wired into no binary** | `internal/domain/scheduling`, `internal/infrastructure/cronstore` (schema v2, `Kind` discriminator), `internal/infrastructure/crondelivery` — all present and green, but `go list -deps` for every `cmd/` contains none of them, and `cron`-expression firing arithmetic is unimplemented (`schedule.go`, \"slice 2\"). `ARCHITECTURE.md` still asserts \"**No daemon-internal cron.**\" |\n| Workflow engines (multi-stage, resumable) | **Implemented, worktree-bound** | `internal/domain/workflow` — every non-triage workflow opens with `StagePrepareWorktree()` and mounts `tc.Dir` into a container; the task persists after every stage (`workflow.go`). `triage.go` documents there is \"no cheaper directory-less classification primitive\" |\n| Long-running chat-initiated work with progress | **Partial** | Chat turns run **once, synchronously**, inline in the channel's turn runner (`internal/gateway/turn.go`). Chat can spawn a *workflow task* (`task_spawn`) which is asynchronous — but that path is worktree/PR-shaped |\n| **Operator-push delivery of an async result** | **Aspirational — zero code** | Nothing pushes a finished result to a chat. No channel subscribes to the event bus (Telegram only *publishes* `KindTurnCompleted`); `crondelivery.Courier` is the intended seam but has **no implementation and no caller**; `send_file` works only inside the live turn. The one out-of-band notification in the tree is feasibility's n8n webhook (`feasibility.go`) |\n| Citation storage / source ledger | **Aspirational — zero code** | No source, citation, or provenance store exists for research output |\n\n\u003e **Target state, not current state.** Everything marked aspirational below is\n\u003e design, not a description of the daemon as it runs.\n\n## Problem\n\nSam's requirement: *\"Deep research should spawn 1 or many subagents for\nexploration or research work. And they should collate all findings, they\nshould use web search/fetch features.\"*\n\nNone of the three primitives that sentence needs can be assembled from what\nships:\n\n1. **Web search does not exist.** `web_fetch` reads a URL you already have. A\n research loop's first act is to *discover* URLs from a query, and there is\n no tool, provider, or config for that — in archie-core or in the shared SDK.\n2. **Nothing can fan out.** `agent.Subagent` can nest *one* nested generation,\n synchronously, and only the PR reviewer calls it. There is no N-way fan-out,\n no bounded concurrency, no shared budget across children, no collation pass.\n `agentloop` explicitly refuses to orchestrate: it runs one mission and the\n caller owns orchestration — and no caller orchestrates.\n3. **A research run outlives a chat turn.** A chat turn is synchronous and\n inline: the operator waits, and the turn's own context window is what\n carries the result. A fan-out over N subagents with web search is minutes to\n tens of minutes of wall clock. It cannot be a turn.\n\nThe interesting part is not the loop. It is where the boundary sits: which\nexisting long-running mechanism this reuses, how citations survive collation\nwhen the thing being collated is model-written prose, and what stops a run\nfrom spending unbounded money.\n\n## What the run actually is\n\nSix stages, each of which must be a *stage* rather than a prompt instruction:\n\n1. **Frame.** One cheap model call turns the request into a research brief: the\n question, 3..N distinct sub-questions, and a per-sub-question budget. Output\n is structured (a capture tool), not prose — the fan-out width comes from\n this step, so it must be a number the code reads, not a sentence.\n2. **Fan out.** One subagent per sub-question, **in parallel**, with a bounded\n concurrency, each running a read-only research toolset (search + fetch +\n read) against its own fresh context window, each required to record its\n findings as *typed source records* before finishing, and each compressing\n what it returns.\n3. **Cross-check.** Deterministic, no model: group by canonical URL and by\n claim, flag single-sourced claims, mark conflicts. See D6.\n4. **Collate.** One model pass over the children's *conclusions and source\n records* — never their transcripts — that writes the report body against the\n pre-reconciled source set.\n5. **Citation pass.** Its own stage: validate every marker in the body resolves\n to a stored source, and every stored source was fetched successfully.\n6. **Render.** The citation list is rendered from the **source store**, not\n from the collator's prose, so a URL can only appear in a report if some\n subagent actually recorded it.\n\nStages 5 and 6 are the load-bearing decision, and they are what the commercial\nproducts' published failure modes point at: a synthesizer that is free to write\ncitations will invent them, and a citation that cannot be traced to a fetch is\nworse than no citation.\n\n## Prior art: what the disclosed designs and the open-source field say\n\nTwo external research passes inform this design — one on the commercial\nproducts (official engineering blogs, model docs, system cards, plus\npeer-reviewed benchmarks), one on open-source deep-research implementations\nread at pinned commits. The findings that changed decisions:\n\n**Only one fan-out unit buys capacity.** Four different units are in use:\nAnthropic fans out *agents with their own context windows*; Google fans out\n*queries*; Perplexity fans out *generated programs*; OpenAI uses pipeline-stage\nagents for query enrichment only. Query fan-out is cheap and buys breadth of\nretrieval; agent fan-out is what buys context capacity. Our step 2 is agent\nfan-out, deliberately, because context isolation is the thing a single GPU\ncannot otherwise buy.\n\n**Nobody collates section-by-section.** The open-source survey found one merge\nprompt over concatenated child summaries (`final_report_generation` in\n`langchain-ai/open_deep_research`), lead-agent synthesis (Anthropic), a\ncode-level reconcile (Perplexity), or single-model continuation. STORM is the\nonly demonstrated section-mapped merge. This confirms D3: one collation pass,\nnot a mapped merge.\n\n**Citations are lost at the merge in 8 of 10 surveyed repos.** Only `lajosdeme/mole`\nassigns citation markers **mechanically in code before the prompt is built** —\nits `internal/output/report.go` carries the comment *\"it cannot introduce a\ncitation, because the citation markers are assigned mechanically before the\nprompt is built\"* — and stores a verbatim `Quote` plus a `QuoteOffset` (byte\noffset into the fetched document) so a citation can be re-verified later.\nClaims whose quote is not found in the source are dropped by code\n(`internal/actors/mine.go`: `\"claim rejected: quote not found in source\"`).\nThis is the strongest available argument for D1, and `QuoteOffset` is worth\nadopting: it makes re-verification cheap and turns \"the model said so\" into\n\"the document says so, here\".\n\n**The failure is measurable and it is at the orchestrator.** An EMNLP 2026\nstudy localising errors across three top-ranked open-source deep-research\nsystems found **84.7% of final-report errors originate at the orchestrator,\nroughly 31% of them hallucinations and the rest citation mistakes**, with\ninformation passing \"through agents like a telephone game\"\n(https://arxiv.org/abs/2608.24306). Anthropic's own mitigations for this are a\nseparate **CitationAgent** pass after synthesis and a filesystem-artefact\npattern — subagents store work externally and pass back lightweight references\nrather than full outputs. Perplexity reached the same conclusion independently.\nBoth are adopted here: D1 (store, not prose) and a citation pass that is its\nown stage.\n\n**A cross-check stage that no vendor has.** Worker A and worker B never compare\nnotes in any disclosed design, and nothing in the published set *resolves*\nconflicts — Perplexity only \"notes\" them. Grouping findings by URL and by claim,\nflagging single-sourced claims, and marking unresolved conflicts is the\ncheapest correctness win available and is where a local implementation can beat\nthe frontier products rather than imitate them.\n\n**Fan-out is expensive in tokens, and on one GPU that is wall clock.** Anthropic\ndiscloses ~15x tokens for multi-agent versus ~4x for a single agent, and\nexplicit tier rules in the orchestrator prompt (1 agent / 3-10 tool calls;\n2-4 subagents / 10-15 calls each; \u003e10 subagents for complex research, 3-5\nspawned concurrently). On a single card, N concurrent workers contend for the\nsame weights and KV budget, so **wall clock scales roughly linearly with worker\ncount** — the isolation benefit survives, the throughput benefit does not. One\ninstrumented study puts **web search at 73% of wall-clock time on average (up to\n91%)** (https://cseweb.ucsd.edu/~yiying/2025_NIPS_ERW_Deep_Research_Perf_Study.pdf),\nwhich is the argument for fanning out *retrieval* aggressively and *generation*\nsparingly, and for overlapping generation with retrieval.\n\n**Frontier citation volume is not a target.** Frontier runs emit 41-113\ncitation URLs per query, while 3-13% of citation URLs are hallucinated and \"more\ncitations per query do not mean fewer errors per citation\"\n(https://arxiv.org/abs/2604.03173). Ten to twenty *verified* citations is a\ndefensible, evidence-backed target.\n\n**Do not port Perplexity's style.** Search-as-code-generation needs\nfrontier-class codegen against a bespoke SDK and an atomised self-owned search\nstack. Port the *intent* — deterministic fan-out, dedupe and filter in code\nbefore anything reaches a model — and leave the code generation to the Go\norchestrator.\n\n## Architecture: three candidate shapes\n\n| # | Shape | Where a run starts | Durability | New code | Cost |\n|---|---|---|---|---|---|\n| **A** | New capability family `internal/domain/research` + `infrastructure/research`, typed engine, owning registry, lifecycle, background run, completion delivered through an injected courier | A chat tool `research_start` returns a run id | Runs survive the turn; progress observable; resumable | Domain + infra + store + wiring + UI + config | Largest |\n| **B** | A fourth cron job kind (`KindResearch`) in `crondelivery`, reusing `scheduling` + `cronstore` + the delivery router | Chat enqueues a one-shot immediate job | Same store and delivery as cron; no new persistence | Runner + kind + blueprint only | Medium — but fakes \"on demand\" as a scheduled job, and a job store keyed on a schedule is the wrong shape for a request |\n| **C** | A `deep_research` **tool** that runs the whole fan-out synchronously inside one chat turn, using `agent.Subagent` for the children | Directly — the model calls the tool | None: the run dies with the turn | Search seam + tool + fan-out helper | Smallest |\n\n**Recommendation: ship C's machinery, then promote it to A.** Slice 2 (below)\nbuilds the orchestrator as a tool because that is the smallest thing that\ndelivers Sam's stated design and reuses the `Subagent` primitive that already\nworks. Slice 3 lifts the identical orchestrator behind a durable run once its\nruntime cost and shape are known from real use — which is exactly the order\nAGENTS.md's solo-project rule asks for (\"prefer the smallest workable change\"),\nand it avoids inventing persistence for a shape that has not been exercised.\n\nShape B is rejected as the primary path: `cronstore.JobSpec` is a *schedule*\n(cron expression, next-run, catch-up); an on-demand research request is not.\nReusing it would mean every ad-hoc research run leaves a permanent job row.\n\n### Where the pieces live (shape C, then A)\n\n| Piece | Home | Contract |\n|---|---|---|\n| Search provider seam | `internal/domain/research` (interface) + `internal/infrastructure/research/search/\u003cprovider\u003e` (impls) | `Searcher.Search(ctx, query, opts) ([]Result, error)`; `Result{URL, Title, Snippet, RetrievedAt}`. Consumer-owned and one-method, per the `crondelivery.Courier` precedent — nothing here talks to a network |\n| `web_search` tool | `internal/tools/search` | Mirrors `internal/tools/webfetch`: `Tool(cfg) *tools.ToolEntry`, returns `nil` when disabled, `ClassIdempotent`, `Toolset: \"web\"` |\n| Source store | `internal/domain/research` entity + store | `Source{ID, URL, CanonicalURL, Title, RetrievedAt, Hash, State(ok/failed/duplicate), CitedBy []subq}` |\n| Fan-out orchestrator | `internal/domain/research` (pure) with the model/tool access injected | Bounded worker pool over sub-questions; per-child `agent.Subagent`-shaped `Researcher` seam; shared step/token/wall-clock budget |\n| Collator | `internal/domain/research` | Model pass over `[]ChildConclusion` + source store → report body |\n| Report renderer | `internal/domain/research` | Markdown + numbered citation list rendered **from the store** |\n| Chat trigger | slice 2: `internal/tools/research`; slice 3: gateway tool + courier in `internal/app/archied` | — |\n| Durable run + delivery (slice 3) | `internal/domain/research` + `internal/infrastructure/research` + `internal/app/archied` wiring | Follows the strict plugin-engine rule in `ARCHITECTURE.md` (\"Plugin engine\nrule (strict)\"): typed contract, owning registry with start/health/stop, narrow typed registrar |\n\n### Why not the curator\n\nThe curator family already looks like the right home — a declared interval, a\ndeclared tool set, an owning registry, per-pass budgets and panic isolation.\nTwo facts rule it out as the host for *this*:\n\n- **Its tool path does not work.** A curator declaring tools is refused\n registration because `ToolBuilder` has no implementation, and even if it were\n bound, `curatorLLMRunner` drops `ChatRequest.Tools` (`main.go`). Fixing\n that is `archie-core-trnd.1`, an independent piece of work.\n- **Its contract is maintenance, not requests.** The isolation invariant\n (`docs/architecture/agent-system.md`, \"Curator isolation invariant\") defines\n curators as periodic peers reasoning over memory. A deep-research run is\n operator-initiated work with a deliverable, on the operator's clock.\n\nA research *curator* is a legitimate later idea — \"research this standing topic\nweekly\" — but it is a consumer of this capability, not its foundation.\n\n## Decisions this design takes\n\n### D1. Citations come from a store, never from prose\n\nEach subagent gets a `record_source` typed tool (the `record_finding` pattern,\n`review.go`) whose handler canonicalises the URL, deduplicates against the\nstore, marks which sub-question recorded it, and appends. The collator may\n*select and order* sources; it may not author them. Rendering the reference\nlist is then a pure function of the store.\n\nThe strongest prior art goes further, and one step of it is worth adopting:\nstore a **verbatim quote plus a byte offset into the fetched document** with\neach claim, and drop any claim whose quote cannot be found in the source\n(`lajosdeme/mole`: `FindQuote` + `Claim.QuoteOffset`, with the miner logging\n`\"claim rejected: quote not found in source\"`). That converts an unfalsifiable\n\"the model said so\" into a re-checkable \"the document says so, at offset N\", and\nit makes citation re-verification cheap rather than a second research run.\n\n*Rejected alternative:* require the child to end with a JSON block of sources\nand parse it. A parse failure loses every source in the child's run, and a\nmodel that reformats its output for readability silently drops citations.\n\n### D2. Fan-out is parallel, bounded, and shares one budget\n\n`agent.Subagent.Run` is synchronous, so N children means N goroutines behind a\nsemaphore — concurrency from config, sized from the VRAM budget rather than\nfrom the research question. Two reasons to keep it small: the shared model\nruntime also serves chat, and on a single GPU N concurrent workers contend for\nthe same weights and KV budget, so **wall clock scales roughly linearly with\nworker count**. The isolation benefit of fan-out survives that; the throughput\nbenefit does not.\n\nFan-out width and per-child `MaxSteps` are enforced by the orchestrator, not\nrequested in a prompt, against a whole-run wall clock and token ceiling.\n`agentloop.Budget` already carries `MaxSteps`, `MaxTokens`, `WallClock`, and\n`TokenBudgetIs(maxTokens)` exists as a stop condition (`agentloop.go`) —\nthe shape exists, the fan-out owner does not.\n\n**Fan out retrieval aggressively and generation sparingly.** Web search is 73%\nof wall-clock time on average (up to 91%) in the one instrumented study\navailable, so retrieval is where the parallel win lives; generation is the\nserialised, expensive half.\n\n### D3. The orchestrator never holds child transcripts\n\n`Subagent.Run` sends a prompt and returns a string; the parent's history is\nnever passed down and the child's is never returned. That is the property that\nmakes N children affordable at all — the parent's context grows by N\nconclusions, not N conversations. This is also why collation is a *separate\npass* rather than an accumulating agent loop: the collator is the only place\nthat must hold all N conclusions, and it holds nothing else.\n\nEach child compresses its own findings before returning, so what crosses the\nboundary is a summary, not a transcript. Anthropic names the same pattern\n(filesystem artefacts with lightweight references back) and Perplexity arrived\nat it independently; it is the one disclosed primitive that *increases*\neffective capacity without a bigger context window.\n\n### D4. Search is a provider seam with no silent default\n\n`web_search` is registered only when a provider resolves, following two\nexisting conventions: return `nil` for a disabled capability\n(`webfetch/tool.go` — \"an advertised tool that never works costs a round\ntrip\") and resolve the credential *before* registering\n(`registerMinimaxTool`, `bootstrap.go`). At least two implementations are\nwanted — a self-hosted meta-search endpoint (no per-query cost, no data egress)\nand one hosted API (quality fallback) — behind the same one-method interface,\nselected by config.\n\n### D5. A run reports what it could not establish\n\nThe collator is required to emit, alongside the report, the sub-questions that\nreturned nothing usable and the sources that failed to fetch. A research report\nthat omits its own gaps reads as authoritative over a hole, and the operator\ncannot tell a negative finding from a missing one.\n\n### D6. Two stages no surveyed design has: a mechanical cross-check and a separate citation pass\n\nChildren never compare notes in any disclosed design, and nothing in the\npublished set *resolves* conflicts — Perplexity only \"notes\" them. Because\n84.7% of final-report errors originate at the orchestrator (mostly citation\nerrors), both gaps are worth closing in code rather than in a prompt:\n\n- **Cross-check (deterministic, no model).** Group findings by canonical URL and\n by claim. Flag single-sourced claims, mark unresolved conflicts explicitly,\n and count sources per sub-question. This runs *before* the collator, so the\n collator receives a pre-reconciled set rather than raw prose to untangle.\n- **Citation pass (its own stage, after collation).** Anthropic runs synthesis\n and then a separate CitationAgent. Here the pass is cheaper for being partly\n mechanical: it validates that every marker in the body resolves to a stored\n source, and that every stored source was actually fetched successfully.\n\nBoth stages are small, both are testable without a model, and together they are\nwhere this implementation can be *better* than the frontier products rather\nthan an imitation of them. A run that is expected to emit 10-20 well-verified\ncitations rather than 41-113 partly-unverified ones is making a deliberate,\nevidence-supported trade.\n\n## Open decisions\n\n1. **Trigger surface.** A chat tool that blocks the turn (shape C) is\n immediately useful but the operator waits; a durable run (shape A) is the\n right end state. Does slice 2 ship the blocking tool, or does the capability\n go straight to durability?\n2. **Delivery for a durable run.** Which channel gets the finished report, and\n is the report an inline message, a file, or a served page? The\n `crondelivery.Courier` seam covers the chat case; a long cited report\n probably wants a document, since chat surfaces render markdown but not\n footnotes well.\n3. **Search provider choice and egress.** A hosted search API sends every query\n to a third party; a self-hosted meta-search keeps it local. This is an\n operator decision with a real privacy dimension, and the default matters.\n4. **Report persistence and reuse.** Does a report become a memory, a document,\n or neither? Reusing a prior report is the difference between \"research\" and\n \"research again next week\".\n5. **Cost ceiling.** What is the per-run token ceiling, and is it a config\n value, a per-request parameter, or derived from the fan-out width?\n\n## Failure and security paths\n\n- **SSRF.** Search results are attacker-influenceable (a page can rank for a\n query). `web_fetch` already vets the *resolved address* at dial time with\n `AllowPrivateNetworks` off, which is the property to preserve — a search\n tool must not become a way to reach the daemon's own control surfaces by\n returning an internal URL.\n- **Prompt injection.** Fetched pages are untrusted content and will contain\n instructions. Subagents fetch; the orchestrator and collator must never\n execute an instruction sourced from a page. Bounded by giving subagents a\n read-only toolset with no shell and no write.\n- **Runaway spend.** Unbounded fan-out × unbounded steps is the failure mode.\n Bounded concurrency, per-child step caps, a whole-run wall clock, and a\n token ceiling, all enforced in code.\n- **Degenerate fan-out.** A model asked for \"N sub-questions\" will produce N\n near-duplicates of the same question. Deduplicate sub-questions before\n dispatch, and require distinct search queries per child.\n- **Failed dependencies.** Search provider down: the tool is absent (D4), not\n broken. Fetch fails: recorded as a `failed` source so the report can say so.\n- **Collator degradation.** If the collator call fails or truncates, the report\n must still be rendered from the source store and the child conclusions —\n degraded, not empty. This mirrors the reviewer's rule that findings captured\n before a failed run are still real (`review.go`).\n\n## Slices\n\n1. **Search seam + `web_search` tool.** Interface, one implementation, config\n block, tool registration, tests. Independently useful to every chat turn.\n2. **Fan-out orchestrator + collation, as a tool.** `record_source` capture\n tool (with quote + offset), bounded parallel children with per-child\n compression, cross-check, collator pass, citation pass, store-rendered\n citations. Triggerable from chat; blocks the turn.\n3. **Durable run + delivery.** Promote the orchestrator behind the engine\n contract, add the run store, progress events, and completion delivery.\n4. **UI + reuse.** Dashboard surface for runs and past reports; optional\n research curator on a schedule.\n\n## Non-goals\n\n- Replacing `web_fetch`; research uses it.\n- A general multi-agent collaboration framework (`archie-core-1786637492113-89`\n owns that question).\n- Report publication to a static site or external destination.\n- Fine-tuning or a dedicated model: the capability is orchestration, and it\n must work on whatever model the operator has configured.\n" + "body": "# Deep research capability — design\n\n**Status:** Draft\n**Date:** 2026-09-14\n**Scope:** A deep-research capability in archie-core: an orchestrator that\nspawns 1..N subagents to explore and research in parallel, collates their\nfindings, and returns one cited report. Grounded in web search and web fetch.\n**Beads issue:** none yet (this document is the input to it)\n**Depends on:** nothing shipped for slice 1; slice 2 depends on slice 1.\n\n## §0 Implementation status\n\nEvery claim in this document is mapped to what exists in the tree.\n\n| Claim | Status | Where it stands |\n|---|---|---|\n| `web_fetch` tool (URL → readable text) | **Implemented** | `internal/tools/webfetch/{tool.go,webfetch.go}`; registered in `internal/app/archied/bootstrap.go` `registerStandaloneTools`, disabled-by-config returns `nil` |\n| **Web search** (query → result list) | **Aspirational — zero code** | No search provider, no search tool, no config block anywhere in the tree or in `ai-sdk` v0.1.32 |\n| `agent.Subagent` — nested agent in a fresh context window | **Implemented** (dependency) | `github.com/samcharles93/ai-sdk@v0.1.32/agent/subagent.go`. Synchronous, non-streaming, own model/system/toolset, `MaxSteps` defaults to 10, nesting depth capped at 5, returns final text only |\n| Subagent used in production | **Implemented, one consumer** | `internal/app/agentworker/review.go` — the adversarial reviewer. That is the only non-test use of `Subagent` in the repo |\n| A **delegate/subagent tool** on the chat surface | **Aspirational — zero code** | No `subagent`/`delegate` tool name is registered anywhere |\n| **Fan-out orchestration** (N missions, one collator) | **Aspirational — zero code** | `agentloop`'s package doc states it runs *one* mission: \"orchestration across missions lives with the caller\". No caller does this yet |\n| Parallel-safe tool execution | **Aspirational — zero code** | The unreachable `internal/tools/dispatch.go` dispatch/gating engine (`ExecutionClass`, `DispatchSequential`/`DispatchConcurrent`/`ClassifyEntry`) was deleted as dead code (tools-core-1); live concurrency is ai-sdk's `GenerateOptions.MaxParallelToolCalls`, which archie-core never sets |\n| Per-result caps + spill-to-disk | **Implemented** | `internal/agentexec/toolset.go` `ToolLimits{MaxResultChars,SpillDir}`, applied in `toolExecute` via `tools.CapPayload` (`limit.go`); no **aggregate** turn cap by design (`toolset.go`). `config.ToolPolicy.SpillDir`; default cap 50k chars, spill dir deliberately not defaulted |\n| Tool guardrails (repeat-failure / no-progress) | **Implemented, not on the call path** | `internal/tools/guardrail.go` engine exists. Its only non-test caller is `dispatchOne` — which is itself unreachable — so **the chat path never consults it**, and the agent path only records per *stage* (`domain/workflow/agent.go`) |\n| Structured capture tools (model reports into a typed store, not prose) | **Implemented** | `internal/app/agentworker/review.go` `record_finding` / `record_checked` — `core.NewTypedTool` appending to a caller-owned slice. This is the pattern a research source ledger should reuse |\n| Curator engine (interval, declared tool set, lifecycle) | **Implemented, tools broken** | `internal/domain/curator`; a curator declaring tools is **refused** at registration (`registry.go`, no `ToolBuilder` implementation) and `curatorLLMRunner` (`internal/app/archied/main.go`) drops `ChatRequest.Tools`. Budgets are time+concurrency only (`runtime.go`) — **no token budget** |\n| Cron / scheduling + delivery router | **Implemented, wired into the daemon** | `internal/domain/scheduling` (ticker engine), `internal/infrastructure/cronstore` (schema v2, `Kind` discriminator), and `internal/infrastructure/crondelivery`; `internal/app/archied/scheduling.go` composes them and `main.go` starts the engine. |\n| Workflow engines (multi-stage, resumable) | **Implemented, worktree-bound** | `internal/domain/workflow` — every non-triage workflow opens with `StagePrepareWorktree()` and mounts `tc.Dir` into a container; the task persists after every stage (`workflow.go`). `triage.go` documents there is \"no cheaper directory-less classification primitive\" |\n| Long-running chat-initiated work with progress | **Partial** | Chat turns run **once, synchronously**, inline in the channel's turn runner (`internal/gateway/turn.go`). Chat can spawn a *workflow task* (`task_spawn`) which is asynchronous — but that path is worktree/PR-shaped |\n| **Operator-push delivery of an async result** | **Aspirational — zero code** | Nothing pushes a finished result to a chat. No channel subscribes to the event bus (Telegram only *publishes* `KindTurnCompleted`); `crondelivery.Courier` is the intended seam but has **no implementation and no caller**; `send_file` works only inside the live turn. The one out-of-band notification in the tree is feasibility's n8n webhook (`feasibility.go`) |\n| Citation storage / source ledger | **Aspirational — zero code** | No source, citation, or provenance store exists for research output |\n\n\u003e **Target state, not current state.** Everything marked aspirational below is\n\u003e design, not a description of the daemon as it runs.\n\n## Problem\n\nSam's requirement: *\"Deep research should spawn 1 or many subagents for\nexploration or research work. And they should collate all findings, they\nshould use web search/fetch features.\"*\n\nNone of the three primitives that sentence needs can be assembled from what\nships:\n\n1. **Web search does not exist.** `web_fetch` reads a URL you already have. A\n research loop's first act is to *discover* URLs from a query, and there is\n no tool, provider, or config for that — in archie-core or in the shared SDK.\n2. **Nothing can fan out.** `agent.Subagent` can nest *one* nested generation,\n synchronously, and only the PR reviewer calls it. There is no N-way fan-out,\n no bounded concurrency, no shared budget across children, no collation pass.\n `agentloop` explicitly refuses to orchestrate: it runs one mission and the\n caller owns orchestration — and no caller orchestrates.\n3. **A research run outlives a chat turn.** A chat turn is synchronous and\n inline: the operator waits, and the turn's own context window is what\n carries the result. A fan-out over N subagents with web search is minutes to\n tens of minutes of wall clock. It cannot be a turn.\n\nThe interesting part is not the loop. It is where the boundary sits: which\nexisting long-running mechanism this reuses, how citations survive collation\nwhen the thing being collated is model-written prose, and what stops a run\nfrom spending unbounded money.\n\n## What the run actually is\n\nSix stages, each of which must be a *stage* rather than a prompt instruction:\n\n1. **Frame.** One cheap model call turns the request into a research brief: the\n question, 3..N distinct sub-questions, and a per-sub-question budget. Output\n is structured (a capture tool), not prose — the fan-out width comes from\n this step, so it must be a number the code reads, not a sentence.\n2. **Fan out.** One subagent per sub-question, **in parallel**, with a bounded\n concurrency, each running a read-only research toolset (search + fetch +\n read) against its own fresh context window, each required to record its\n findings as *typed source records* before finishing, and each compressing\n what it returns.\n3. **Cross-check.** Deterministic, no model: group by canonical URL and by\n claim, flag single-sourced claims, mark conflicts. See D6.\n4. **Collate.** One model pass over the children's *conclusions and source\n records* — never their transcripts — that writes the report body against the\n pre-reconciled source set.\n5. **Citation pass.** Its own stage: validate every marker in the body resolves\n to a stored source, and every stored source was fetched successfully.\n6. **Render.** The citation list is rendered from the **source store**, not\n from the collator's prose, so a URL can only appear in a report if some\n subagent actually recorded it.\n\nStages 5 and 6 are the load-bearing decision, and they are what the commercial\nproducts' published failure modes point at: a synthesizer that is free to write\ncitations will invent them, and a citation that cannot be traced to a fetch is\nworse than no citation.\n\n## Prior art: what the disclosed designs and the open-source field say\n\nTwo external research passes inform this design — one on the commercial\nproducts (official engineering blogs, model docs, system cards, plus\npeer-reviewed benchmarks), one on open-source deep-research implementations\nread at pinned commits. The findings that changed decisions:\n\n**Only one fan-out unit buys capacity.** Four different units are in use:\nAnthropic fans out *agents with their own context windows*; Google fans out\n*queries*; Perplexity fans out *generated programs*; OpenAI uses pipeline-stage\nagents for query enrichment only. Query fan-out is cheap and buys breadth of\nretrieval; agent fan-out is what buys context capacity. Our step 2 is agent\nfan-out, deliberately, because context isolation is the thing a single GPU\ncannot otherwise buy.\n\n**Nobody collates section-by-section.** The open-source survey found one merge\nprompt over concatenated child summaries (`final_report_generation` in\n`langchain-ai/open_deep_research`), lead-agent synthesis (Anthropic), a\ncode-level reconcile (Perplexity), or single-model continuation. STORM is the\nonly demonstrated section-mapped merge. This confirms D3: one collation pass,\nnot a mapped merge.\n\n**Citations are lost at the merge in 8 of 10 surveyed repos.** Only `lajosdeme/mole`\nassigns citation markers **mechanically in code before the prompt is built** —\nits `internal/output/report.go` carries the comment *\"it cannot introduce a\ncitation, because the citation markers are assigned mechanically before the\nprompt is built\"* — and stores a verbatim `Quote` plus a `QuoteOffset` (byte\noffset into the fetched document) so a citation can be re-verified later.\nClaims whose quote is not found in the source are dropped by code\n(`internal/actors/mine.go`: `\"claim rejected: quote not found in source\"`).\nThis is the strongest available argument for D1, and `QuoteOffset` is worth\nadopting: it makes re-verification cheap and turns \"the model said so\" into\n\"the document says so, here\".\n\n**The failure is measurable and it is at the orchestrator.** An EMNLP 2026\nstudy localising errors across three top-ranked open-source deep-research\nsystems found **84.7% of final-report errors originate at the orchestrator,\nroughly 31% of them hallucinations and the rest citation mistakes**, with\ninformation passing \"through agents like a telephone game\"\n(https://arxiv.org/abs/2608.24306). Anthropic's own mitigations for this are a\nseparate **CitationAgent** pass after synthesis and a filesystem-artefact\npattern — subagents store work externally and pass back lightweight references\nrather than full outputs. Perplexity reached the same conclusion independently.\nBoth are adopted here: D1 (store, not prose) and a citation pass that is its\nown stage.\n\n**A cross-check stage that no vendor has.** Worker A and worker B never compare\nnotes in any disclosed design, and nothing in the published set *resolves*\nconflicts — Perplexity only \"notes\" them. Grouping findings by URL and by claim,\nflagging single-sourced claims, and marking unresolved conflicts is the\ncheapest correctness win available and is where a local implementation can beat\nthe frontier products rather than imitate them.\n\n**Fan-out is expensive in tokens, and on one GPU that is wall clock.** Anthropic\ndiscloses ~15x tokens for multi-agent versus ~4x for a single agent, and\nexplicit tier rules in the orchestrator prompt (1 agent / 3-10 tool calls;\n2-4 subagents / 10-15 calls each; \u003e10 subagents for complex research, 3-5\nspawned concurrently). On a single card, N concurrent workers contend for the\nsame weights and KV budget, so **wall clock scales roughly linearly with worker\ncount** — the isolation benefit survives, the throughput benefit does not. One\ninstrumented study puts **web search at 73% of wall-clock time on average (up to\n91%)** (https://cseweb.ucsd.edu/~yiying/2025_NIPS_ERW_Deep_Research_Perf_Study.pdf),\nwhich is the argument for fanning out *retrieval* aggressively and *generation*\nsparingly, and for overlapping generation with retrieval.\n\n**Frontier citation volume is not a target.** Frontier runs emit 41-113\ncitation URLs per query, while 3-13% of citation URLs are hallucinated and \"more\ncitations per query do not mean fewer errors per citation\"\n(https://arxiv.org/abs/2604.03173). Ten to twenty *verified* citations is a\ndefensible, evidence-backed target.\n\n**Do not port Perplexity's style.** Search-as-code-generation needs\nfrontier-class codegen against a bespoke SDK and an atomised self-owned search\nstack. Port the *intent* — deterministic fan-out, dedupe and filter in code\nbefore anything reaches a model — and leave the code generation to the Go\norchestrator.\n\n## Architecture: three candidate shapes\n\n| # | Shape | Where a run starts | Durability | New code | Cost |\n|---|---|---|---|---|---|\n| **A** | New capability family `internal/domain/research` + `infrastructure/research`, typed engine, owning registry, lifecycle, background run, completion delivered through an injected courier | A chat tool `research_start` returns a run id | Runs survive the turn; progress observable; resumable | Domain + infra + store + wiring + UI + config | Largest |\n| **B** | A fourth cron job kind (`KindResearch`) in `crondelivery`, reusing `scheduling` + `cronstore` + the delivery router | Chat enqueues a one-shot immediate job | Same store and delivery as cron; no new persistence | Runner + kind + blueprint only | Medium — but fakes \"on demand\" as a scheduled job, and a job store keyed on a schedule is the wrong shape for a request |\n| **C** | A `deep_research` **tool** that runs the whole fan-out synchronously inside one chat turn, using `agent.Subagent` for the children | Directly — the model calls the tool | None: the run dies with the turn | Search seam + tool + fan-out helper | Smallest |\n\n**Recommendation: ship C's machinery, then promote it to A.** Slice 2 (below)\nbuilds the orchestrator as a tool because that is the smallest thing that\ndelivers Sam's stated design and reuses the `Subagent` primitive that already\nworks. Slice 3 lifts the identical orchestrator behind a durable run once its\nruntime cost and shape are known from real use — which is exactly the order\nAGENTS.md's solo-project rule asks for (\"prefer the smallest workable change\"),\nand it avoids inventing persistence for a shape that has not been exercised.\n\nShape B is rejected as the primary path: `cronstore.JobSpec` is a *schedule*\n(cron expression, next-run, catch-up); an on-demand research request is not.\nReusing it would mean every ad-hoc research run leaves a permanent job row.\n\n### Where the pieces live (shape C, then A)\n\n| Piece | Home | Contract |\n|---|---|---|\n| Search provider seam | `internal/domain/research` (interface) + `internal/infrastructure/research/search/\u003cprovider\u003e` (impls) | `Searcher.Search(ctx, query, opts) ([]Result, error)`; `Result{URL, Title, Snippet, RetrievedAt}`. Consumer-owned and one-method, per the `crondelivery.Courier` precedent — nothing here talks to a network |\n| `web_search` tool | `internal/tools/search` | Mirrors `internal/tools/webfetch`: `Tool(cfg) *tools.ToolEntry`, returns `nil` when disabled, `ClassIdempotent`, `Toolset: \"web\"` |\n| Source store | `internal/domain/research` entity + store | `Source{ID, URL, CanonicalURL, Title, RetrievedAt, Hash, State(ok/failed/duplicate), CitedBy []subq}` |\n| Fan-out orchestrator | `internal/domain/research` (pure) with the model/tool access injected | Bounded worker pool over sub-questions; per-child `agent.Subagent`-shaped `Researcher` seam; shared step/token/wall-clock budget |\n| Collator | `internal/domain/research` | Model pass over `[]ChildConclusion` + source store → report body |\n| Report renderer | `internal/domain/research` | Markdown + numbered citation list rendered **from the store** |\n| Chat trigger | slice 2: `internal/tools/research`; slice 3: gateway tool + courier in `internal/app/archied` | — |\n| Durable run + delivery (slice 3) | `internal/domain/research` + `internal/infrastructure/research` + `internal/app/archied` wiring | Follows the strict plugin-engine rule in `docs/architecture/plugins-and-extensions.md` (\"Plugin engine\nrule (strict)\"): typed contract, owning registry with start/health/stop, narrow typed registrar |\n\n### Why not the curator\n\nThe curator family already looks like the right home — a declared interval, a\ndeclared tool set, an owning registry, per-pass budgets and panic isolation.\nTwo facts rule it out as the host for *this*:\n\n- **Its tool path does not work.** A curator declaring tools is refused\n registration because `ToolBuilder` has no implementation, and even if it were\n bound, `curatorLLMRunner` drops `ChatRequest.Tools` (`main.go`). Fixing\n that is `archie-core-trnd.1`, an independent piece of work.\n- **Its contract is maintenance, not requests.** The isolation invariant\n (`docs/architecture/agent-system.md`, \"Curator isolation invariant\") defines\n curators as periodic peers reasoning over memory. A deep-research run is\n operator-initiated work with a deliverable, on the operator's clock.\n\nA research *curator* is a legitimate later idea — \"research this standing topic\nweekly\" — but it is a consumer of this capability, not its foundation.\n\n## Decisions this design takes\n\n### D1. Citations come from a store, never from prose\n\nEach subagent gets a `record_source` typed tool (the `record_finding` pattern,\n`review.go`) whose handler canonicalises the URL, deduplicates against the\nstore, marks which sub-question recorded it, and appends. The collator may\n*select and order* sources; it may not author them. Rendering the reference\nlist is then a pure function of the store.\n\nThe strongest prior art goes further, and one step of it is worth adopting:\nstore a **verbatim quote plus a byte offset into the fetched document** with\neach claim, and drop any claim whose quote cannot be found in the source\n(`lajosdeme/mole`: `FindQuote` + `Claim.QuoteOffset`, with the miner logging\n`\"claim rejected: quote not found in source\"`). That converts an unfalsifiable\n\"the model said so\" into a re-checkable \"the document says so, at offset N\", and\nit makes citation re-verification cheap rather than a second research run.\n\n*Rejected alternative:* require the child to end with a JSON block of sources\nand parse it. A parse failure loses every source in the child's run, and a\nmodel that reformats its output for readability silently drops citations.\n\n### D2. Fan-out is parallel, bounded, and shares one budget\n\n`agent.Subagent.Run` is synchronous, so N children means N goroutines behind a\nsemaphore — concurrency from config, sized from the VRAM budget rather than\nfrom the research question. Two reasons to keep it small: the shared model\nruntime also serves chat, and on a single GPU N concurrent workers contend for\nthe same weights and KV budget, so **wall clock scales roughly linearly with\nworker count**. The isolation benefit of fan-out survives that; the throughput\nbenefit does not.\n\nFan-out width and per-child `MaxSteps` are enforced by the orchestrator, not\nrequested in a prompt, against a whole-run wall clock and token ceiling.\n`agentloop.Budget` already carries `MaxSteps`, `MaxTokens`, `WallClock`, and\n`TokenBudgetIs(maxTokens)` exists as a stop condition (`agentloop.go`) —\nthe shape exists, the fan-out owner does not.\n\n**Fan out retrieval aggressively and generation sparingly.** Web search is 73%\nof wall-clock time on average (up to 91%) in the one instrumented study\navailable, so retrieval is where the parallel win lives; generation is the\nserialised, expensive half.\n\n### D3. The orchestrator never holds child transcripts\n\n`Subagent.Run` sends a prompt and returns a string; the parent's history is\nnever passed down and the child's is never returned. That is the property that\nmakes N children affordable at all — the parent's context grows by N\nconclusions, not N conversations. This is also why collation is a *separate\npass* rather than an accumulating agent loop: the collator is the only place\nthat must hold all N conclusions, and it holds nothing else.\n\nEach child compresses its own findings before returning, so what crosses the\nboundary is a summary, not a transcript. Anthropic names the same pattern\n(filesystem artefacts with lightweight references back) and Perplexity arrived\nat it independently; it is the one disclosed primitive that *increases*\neffective capacity without a bigger context window.\n\n### D4. Search is a provider seam with no silent default\n\n`web_search` is registered only when a provider resolves, following two\nexisting conventions: return `nil` for a disabled capability\n(`webfetch/tool.go` — \"an advertised tool that never works costs a round\ntrip\") and resolve the credential *before* registering\n(`registerMinimaxTool`, `bootstrap.go`). At least two implementations are\nwanted — a self-hosted meta-search endpoint (no per-query cost, no data egress)\nand one hosted API (quality fallback) — behind the same one-method interface,\nselected by config.\n\n### D5. A run reports what it could not establish\n\nThe collator is required to emit, alongside the report, the sub-questions that\nreturned nothing usable and the sources that failed to fetch. A research report\nthat omits its own gaps reads as authoritative over a hole, and the operator\ncannot tell a negative finding from a missing one.\n\n### D6. Two stages no surveyed design has: a mechanical cross-check and a separate citation pass\n\nChildren never compare notes in any disclosed design, and nothing in the\npublished set *resolves* conflicts — Perplexity only \"notes\" them. Because\n84.7% of final-report errors originate at the orchestrator (mostly citation\nerrors), both gaps are worth closing in code rather than in a prompt:\n\n- **Cross-check (deterministic, no model).** Group findings by canonical URL and\n by claim. Flag single-sourced claims, mark unresolved conflicts explicitly,\n and count sources per sub-question. This runs *before* the collator, so the\n collator receives a pre-reconciled set rather than raw prose to untangle.\n- **Citation pass (its own stage, after collation).** Anthropic runs synthesis\n and then a separate CitationAgent. Here the pass is cheaper for being partly\n mechanical: it validates that every marker in the body resolves to a stored\n source, and that every stored source was actually fetched successfully.\n\nBoth stages are small, both are testable without a model, and together they are\nwhere this implementation can be *better* than the frontier products rather\nthan an imitation of them. A run that is expected to emit 10-20 well-verified\ncitations rather than 41-113 partly-unverified ones is making a deliberate,\nevidence-supported trade.\n\n## Open decisions\n\n1. **Trigger surface.** A chat tool that blocks the turn (shape C) is\n immediately useful but the operator waits; a durable run (shape A) is the\n right end state. Does slice 2 ship the blocking tool, or does the capability\n go straight to durability?\n2. **Delivery for a durable run.** Which channel gets the finished report, and\n is the report an inline message, a file, or a served page? The\n `crondelivery.Courier` seam covers the chat case; a long cited report\n probably wants a document, since chat surfaces render markdown but not\n footnotes well.\n3. **Search provider choice and egress.** A hosted search API sends every query\n to a third party; a self-hosted meta-search keeps it local. This is an\n operator decision with a real privacy dimension, and the default matters.\n4. **Report persistence and reuse.** Does a report become a memory, a document,\n or neither? Reusing a prior report is the difference between \"research\" and\n \"research again next week\".\n5. **Cost ceiling.** What is the per-run token ceiling, and is it a config\n value, a per-request parameter, or derived from the fan-out width?\n\n## Failure and security paths\n\n- **SSRF.** Search results are attacker-influenceable (a page can rank for a\n query). `web_fetch` already vets the *resolved address* at dial time with\n `AllowPrivateNetworks` off, which is the property to preserve — a search\n tool must not become a way to reach the daemon's own control surfaces by\n returning an internal URL.\n- **Prompt injection.** Fetched pages are untrusted content and will contain\n instructions. Subagents fetch; the orchestrator and collator must never\n execute an instruction sourced from a page. Bounded by giving subagents a\n read-only toolset with no shell and no write.\n- **Runaway spend.** Unbounded fan-out × unbounded steps is the failure mode.\n Bounded concurrency, per-child step caps, a whole-run wall clock, and a\n token ceiling, all enforced in code.\n- **Degenerate fan-out.** A model asked for \"N sub-questions\" will produce N\n near-duplicates of the same question. Deduplicate sub-questions before\n dispatch, and require distinct search queries per child.\n- **Failed dependencies.** Search provider down: the tool is absent (D4), not\n broken. Fetch fails: recorded as a `failed` source so the report can say so.\n- **Collator degradation.** If the collator call fails or truncates, the report\n must still be rendered from the source store and the child conclusions —\n degraded, not empty. This mirrors the reviewer's rule that findings captured\n before a failed run are still real (`review.go`).\n\n## Slices\n\n1. **Search seam + `web_search` tool.** Interface, one implementation, config\n block, tool registration, tests. Independently useful to every chat turn.\n2. **Fan-out orchestrator + collation, as a tool.** `record_source` capture\n tool (with quote + offset), bounded parallel children with per-child\n compression, cross-check, collator pass, citation pass, store-rendered\n citations. Triggerable from chat; blocks the turn.\n3. **Durable run + delivery.** Promote the orchestrator behind the engine\n contract, add the run store, progress events, and completion delivery.\n4. **UI + reuse.** Dashboard surface for runs and past reports; optional\n research curator on a schedule.\n\n## Non-goals\n\n- Replacing `web_fetch`; research uses it.\n- A general multi-agent collaboration framework (`archie-core-1786637492113-89`\n owns that question).\n- Report publication to a static site or external destination.\n- Fine-tuning or a dedicated model: the capability is orchestration, and it\n must work on whatever model the operator has configured.\n" }, { "url": "/dev/prds/dynamic-curators/", @@ -220,7 +220,7 @@ "url": "/dev/prds/eda-playbook-engine/", "title": "EDA playbook engine -- design", "section": "PRDs", - "body": "# EDA playbook engine -- design\n\n**Status:** Draft\n**Date:** 2026-09-03\n**Beads epic:** `archie-core-t2db`\n\n## Problem\n\n\"Which workflow runs for which trigger\" is hardcoded Go: a literal\nlabel vocabulary (`workintake.labelKinds`) and a literal label-\u003eworkflow map\n(`workflow/routing.go`), extended only by adding a Go case or shipping a\nskill whose `metadata.archie.workflow` overrides one *named* built-in\nworkflow. Extending the system to a new trigger, a new action, or a new\nevent source means a feature branch, a code change in a hardcoded\npath, and new Go tests -- for every addition. That is the barrier this\ndocument exists to remove.\n\nThe goal: triggers, the actions they run, and the\nconditions that connect them should be **drop-in and load**, not **drop-in\nand touch application code**. Yaegi-evaluated Go is the unit of imperative\n\"do a thing\" logic; YAML playbooks are pure orchestration data -- no logic,\njust \"on this event, with this data, call these actions, in this order,\nunder these conditions.\" The daemon's event coordinator doesn't know what a\n`bugfix` or `image-gen` action does, it resolves the name against a\nregistry and pipes typed data through.\n\nThis is explicitly **not** scoped to code-generation workflows. A playbook\naction can be any registered capability -- run a codegen workflow, send a\nmessage on a channel, open something on a forge, generate an image -- so\nlong as it's implemented behind one of the typed engine families below.\n\n## Relationship to existing decisions\n\n- `docs/prds/event-sources-and-reactions.md` ruled\n that reactions in the two epics it covers are **producer-only** (no veto,\n no mutation of in-flight work) and explicitly rejected a single generic\n `Source` interface as \"the untyped-hook shape the plugin engine rule\n exists to prevent.\" **This document extends that scope**: playbooks can\n dispatch to multiple typed action kinds, not just \"produce a\n `TaskEnvelope`.\" It does not relitigate producer-only-ness (still no\n veto/mutation of in-flight work here) and it does not introduce one\n generic interface -- see \"Typed positions, not one generic hook\" below,\n which is how it stays compliant with the plugin engine rule instead of\n reopening that document's #4.\n- `ARCHITECTURE.md#plugin-engine-rule-strict` governs every extension point\n introduced here. Every position below is a typed domain contract with an\n owning registry, not a generic callback map. This is a hard constraint,\n confirmed against the existing rule text before writing this doc.\n- Yaegi is already a proven in-process extension mechanism\n (`internal/plugin`, `internal/domain/workflow/wfeval/yaegi.go`,\n `internal/gate/gateeval/yaegi.go`, `internal/skill/plugin.go`) -- this\n design generalizes its existing role, it does not introduce a new\n execution mechanism.\n- `internal/domain/workflow/skillbuild.BuildRegistry` already composes a\n `workflow.Registry` from a skills catalog, plugin-defined overriding\n built-in. The playbook loader described here supersedes label routing\n specifically; it does not replace `skillbuild`'s workflow-composition role,\n it becomes an additional input the daemon composes at startup.\n- Webhook event intake is a separate design. This\n document defines the router each webhook-sourced event should be able to\n reach; it does not redesign webhook receipt itself.\n\n## Shape: four logic positions, each a typed engine family\n\nEvery entrypoint kind (mechanism, action, loop, schedule, route, trigger) is itself a schema emitter: defining a new kind produces the\ngenerated contract downstream tooling consumes. The kinds slot into a fixed\nhierarchy of logic positions:\n\n```\nWorkflow -\u003e Module -\u003e Channel (telegram/whatsapp/email/...)\n -\u003e Forge (GitHub/Gitea/GitLab/...)\n```\n\nEach position is its own typed engine family per the plugin engine rule:\nidentity + capability-specific operations, an owning `Registry`/`Manager`,\nexplicit lifecycle where the implementation owns resources, narrow host\naccess. Concretely:\n\n- **Workflow** -- already exists (`internal/domain/workflow`). A playbook\n action of kind `workflow` invokes a registered `workflow.Registry` entry\n by name, unchanged from the existing execution path.\n- **Module** -- the new general-purpose action position. A `Module` is a\n Yaegi-evaluated implementation of a generated interface (args in, typed\n result out) for one declared action kind (e.g. `image-gen`, `notify`,\n `open-pr-comment`). This is the position that makes actions \"not\n restricted to code-generation.\" Full design (schema-per-kind, the\n registry shape, the trust boundary, the recommended first slice):\n `docs/prds/module-position.md`.\n- **Channel** -- already exists as `channels.Channel` (telegram, email,\n webhook). A playbook action of kind `channel` sends through a named,\n already-registered channel. No new mechanism; this document just makes it\n addressable from a playbook.\n- **Forge** -- forge clients (GitHub, Gitea, GitLab) become an addressable\n action position the same way Channel is, so a playbook can say \"comment\n on this forge issue\" without that being a special case wired only into\n the poller.\n\n**Typed positions, not one generic hook.** There is no single `Action`\ninterface with an `any` payload. Each position keeps its own typed\ncontract; the playbook YAML names a position and an implementation, and the\nevent coordinator resolves that pair against the matching typed registry.\nThis is what keeps the design compliant with the plugin engine rule and\nwith `event-sources-and-reactions.md`'s rejection of a generic `Source`/\nhook interface -- the generality lives in the YAML data model (which\npositions exist, which named implementations are registered), not in a\nruntime `any`-typed dispatch path.\n\n## `go:generate`-driven schemas, from day one\n\nSchemas are not a future nicety bolted onto an ad hoc\nYAML shape. They are the struct+interface contracts for each logic\nposition, and they drive code generation from the start.\n\n- Each logic position (Workflow/Module/Channel/Forge) has a schema source\n (Go-annotated types, following the existing pattern elsewhere in the\n repo) from which `go:generate` emits the interface and struct scaffolding\n an implementation must satisfy.\n- A Yaegi module implements the generated interface for its position. The\n generated contract is what makes an implementation valid, not hand-review.\n- The schema is also what the linter and LSP (below) validate a playbook\n YAML against -- one source of truth for codegen, startup loading, and\n authoring-time tooling, so the three can't drift from each other.\n- Schema versioning is explicit per position, independent of a playbook's\n own workflow version (see collision handling below).\n\nThe generator itself is not designed here. The constraint is\nschema-first, `go:generate`-driven, per logic position.\n\n## Playbook YAML: pure orchestration data\n\nA playbook declares no embedded logic -- only:\n\n- **trigger**: the event predicate (e.g. forge issue assigned + labelled\n `bugfix`, a schedule tick, a webhook payload shape).\n- **actions**: an ordered list, each naming a logic position and a\n registered implementation, with typed argument data and optional\n conditions.\n- **data flow between actions**: results/facts from one action available to\n later actions in the same playbook run -- exact interpolation syntax is\n an open point, not decided here (see Open questions).\n\nPlaybooks live in a configured directory (repo/directory-scoped, per Sam's\n\"define repos, directories, playbooks\" framing) and are loaded at archied\nstartup, alongside the existing skills-catalog composition.\n\n## Dedup: detect and log, no arbitration mechanism\n\nNo runtime auto-resolution rule. An automatic \"pick a winner\" algorithm\n(e.g. by version) is itself a mechanism that can fail and need debugging\nwhen a collision happens -- the wrong kind of complexity to introduce for\na case that should just be visible and fixed by a human. Instead:\n\n- **At daemon startup:** if independently-sourced playbook directories\n declare a collision on the same trigger, that is a definition failure.\n The colliding definitions are dropped (not loaded) and the failure is\n logged plainly (which sources, which trigger) and reported back to the\n caller that requested the load -- not swallowed as a background log line\n only. The schema is the definition; a schema conflict is a reported\n failure, not something the daemon silently arbitrates.\n- **Linter**: a CI/dev-time tool, reading the same generated schemas, that\n flags duplicate or conflicting trigger definitions *within* a single\n maintained tree before merge.\n\n **Resolved** (`archie-core-t2db.12`). A standalone\n gopls-shaped binary (`cmd/archie-playbooks`, lint subcommand) validates\n playbook directories against the exact loaders the daemon uses\n (`LoadPlaybookDirs`/`LoadKindWorkflowsYAML`/`LoadLabelWorkflowsYAML`),\n exiting non-zero on any collision / malformed file / invalid binding.\n Findings are file-granular, not line-granular: the loader decodes with\n `yaml.Unmarshal` into a plain map, which discards line numbers. A\n compiler-style file:line diagnostic needs a `yaml.Node` decoding\n upgrade, tracked separately -- the linter agrees with runtime\n validation by construction, which is the load-bearing property.\n Discoverable via `task lint:playbooks` or direct `go run\n ./cmd/archie-playbooks lint -dir ...`. The LSP/serve mode is a later\n entrypoint of the SAME binary, per the shared-package decision above.\n- **LSP**: same schema source again, feeding author-time hover/completion/\n inline errors while someone edits a playbook YAML in an editor. Kept\n explicitly in scope per Sam (not deferred), because it's the same schema\n artifact as the linter's, just a different consumer.\n\nThe startup log-and-refuse path exists for what the linter/LSP structurally\ncannot see -- sources composed together only at daemon startup, never\nchecked against each other by either tool. If the linter and LSP do the job\nthey're built for, a real collision reaching that startup path should be\nrare in practice, not a routine occurrence the design leans on.\n\n**Shared package, thin consumers -- not a bundled-vs-standalone choice.**\nThe validation logic (`LoadKindWorkflowsYAML`, `LoadLabelWorkflowsYAML`, the\ndirectory-merge from t2db.11) lives in `internal/domain/workflow`, a plain\nGo package with no knowledge of which binary calls it. This makes \"should\nthe linter/LSP be bundled into archied or a separate tool\" a non-question:\n- `archied` already imports the package directly and validates in-process\n at startup -- no subprocess, no external tool call.\n- An agent that needs to validate a playbook in-process can import the same\n package directly, for the same reason.\n- The standalone `cmd/` binary (t2db.12, gopls-shaped: one binary, a lint\n mode now, an LSP/serve mode later) exists only because CI and editors are\n out-of-process consumers by nature -- a CI step needs an exit code, an\n editor speaks LSP over a process boundary, neither can `import` a Go\n package. It is a thin wrapper per `organisation.md`'s `cmd/` rule, not a\n second implementation.\n\nFolding CLI/LSP protocol handling into `archied` itself would grow the\ndaemon into unrelated concerns, against the plugin-engine rule's spirit of\nnarrow capability families.\n\n## Execution-time gaps (both resolved)\n\nEverything above covers *loading* a playbook. Nothing above covers what\nhappens while one *runs*. Two gaps, both load-bearing enough to resolve\nbefore implementation, not defer:\n\n### 1. Mid-playbook action failure\n\n**Resolved** (`archie-core-t2db.18`). The default below\nis a committed decision, not a candidate list; changing it later would\nrequire its own decision. Gap 2 below is resolved by `archie-core-t2db.17`.\n\nA playbook is an ordered action list with data flowing between steps. If\naction 2 of 4 fails at runtime (an image-gen API times out, a forge call\n403s), the run needs a defined outcome. This\ncodebase already answers a structurally identical question for its\nexisting workflow engine, precisely enough to reuse rather than\nre-derive: `workflow.Run` (`internal/domain/workflow/workflow.go`)\ndraws a three-way distinction on a stage failure, and a playbook run\ndraws the same one on an action failure.\n\n**Decision: reuse `Run`'s three-way distinction directly.** The stated\ndefault is **stop-on-first-failure**: a real action error halts the\nplaybook run immediately and is reported, with no continue-to-next-action\nand no automatic retry or compensating follow-up. The three points below\nare the complete decision.\n\n1. **A real action error** (the action's own logic failed -- an API\n error, a validation failure inside the action) stops the run\n immediately. No later actions execute. The failure is reported the\n same way a load collision is reported -- dropped, logged plainly\n (which action and why), and returned to the caller, never swallowed\n as a background log line only (the Dedup section's drop-and-report\n rule). `Run` is the mechanical\n precedent for the stop itself: a stage error sets the park reason and\n returns without running further stages (`workflow.go`). This\n is stop-on-first-failure, matching the document's existing bias\n against clever automatic recovery, and it is now a stated decision\n rather than a nobody-chose default.\n2. **Interruption** (the daemon is shutting down mid-dispatch, `ctx.Err()\n != nil`) is explicitly **not** a failure. `Run` already treats this\n case specially -- \"Daemon shutdown is not a workflow failure. Leave\n the task running ...; parking here would publish a false failure and\n require manual intervention\" (`workflow.go`) -- and the same\n reasoning applies to a playbook run: an in-flight dispatch interrupted\n by restart must not be recorded as a broken playbook. What \"leave it\n alone for restart\" means for a playbook run (there is no `store.Task`\n row a playbook run inherently owns the way a workflow stage does) is\n an open implementation detail for whoever builds the coordinator, not\n decided here -- but the *semantic* (interruption ≠ failure) is decided.\n3. **A run that produces no outcome at all** (every action ran without\n error, but nothing was assigned to happen) is itself an error\n condition, matching `Run`'s own \"a workflow must end with an explicit\n outcome; not doing so is a definition bug, which still must not vanish\n silently\" (`workflow.go`). A playbook with actions that all ran\n but never reached a coherent terminal state is a definition bug in the\n playbook, reported as such.\n\n**Rollback:** explicitly **not attempted**, for exactly the reason the\ngap's own description named -- a posted forge comment is not revocable.\nThis is not a limitation to work around later; it is the correct\nsemantic. An operator who needs a corrective action after a failure\nwrites a *new* playbook/action for that, they do not get automatic\nundo.\n\n**This stays inside the producer-only rule.** Stop-on-first-failure is the\nrun deciding its own terminal state from its own action's error. Nothing\nalready in flight is blocked or altered; the run only declines to start\nactions that had not started yet, and it records the failure on its own\nunit the way `Run` sets a park reason and transitions the task to\n`StatusParked`. A future compensating action (\"if the notify action\nfailed, also log an incident\") would be producing new work, which the rule\nalready permits. No syntax for it is designed here.\n\n**J3 relationship, stated explicitly:** J3 (a CEL `when` condition\nerroring at eval time → treated as false → skip that action → run\ncontinues) is a **different case from an action failure** and is not\nrevisited by this resolution. A condition error means \"we couldn't\ndetermine whether to run this action,\" which this document already\nresolved as \"then don't, and move on.\" An action failure means \"we tried\nto run this action and it broke,\" which stops the run per point 1 above.\nThe two must not be conflated: a coordinator implementation should be\nable to point at one `when`-evaluation code path and one\naction-invocation code path and show they use different outcomes.\n\n### 2. Idempotency at execution time\n\n**Resolved** (`archie-core-t2db.17`, a\ndesign-investigation ticket: resolve, do not implement). The decision\nbelow is the answer; the keying scheme itself is separate follow-up work\nand is intentionally not built here. It **extends**\n`event-sources-and-reactions.md`'s existing contract -- it does not\ncontradict it (relationship argued at the end).\n\n**Problem (unchanged):** `event-sources-and-reactions.md` decided\nreactions ride `internal/eventbus`'s at-least-once delivery specifically\nso `PublishUnique`/idempotency keys absorb redelivery without duplicate\nside effects. This document inherits that for free only for\n`workflow`-kind actions, because `TaskEnvelope` already carries an\nidempotency key. It said nothing for `channel`, `forge`, or `module`\nactions with real side effects -- a posted comment, a sent message, a\nbilled image generation.\n\nOne correction to the problem statement, checked against the code rather\nthan assumed, because it changes what the answer must be:\n**`PublishUnique`'s dedup is not a consumer-redelivery guard at all.**\n`PublishUnique` only sets the JetStream `Nats-Msg-Id` header on the\noutgoing message (`internal/infrastructure/eventbus/nats/publisher.go`;\nkey constant `message.go`); JetStream suppresses *republished*\nmessages carrying a repeated key inside `Config.DedupWindow`\n(`client.go`, `Duplicates: cfg.DedupWindow`), default 2 minutes\n(`config.go`, `DefaultDedupWindow = 2 * time.Minute`). A consumer\nredelivery is the same *stored* message re-delivered on Nak or\nacknowledge-timeout (`internal/eventbus/eventbus.go`: \"a handler\nreturning an error causes redelivery\") -- not a republish; its\n`Nats-Msg-Id` is never re-firewalled, and even if it were, two minutes is\nnot an at-most-once guarantee for a side effect that must never repeat.\n**Conclusion: side-effecting action kinds need a consumer-side, durable\ndedup record; the bus cannot supply it.** That is precisely the situation\n`binding_dispatches` was built for -- so the decision below reuses its\nmechanism rather than designing a new one.\n\n#### Decision\n\n**Granularity: per-action-per-event -- one dedup record per\n(action, event) pair, not per playbook run.** Whole-run keying, the\nobvious alternative, has two crash windows: the run row is written when the run starts, so a crash mid-run\nleaves the run \"started\" and a redelivery skips *every remaining\naction* (work silently lost); a crash before the row write re-runs\n*every already-fired action* (duplicate side effects -- the exact harm\nthis gap exists to prevent). Per-action keys collapse both into one\nwindow and one behavior: an action's record is written immediately\nbefore that action's side effect is invoked, so an action fires iff its\nown record is absent, no fired action ever re-fires, and a redelivered\nevent resumes at the first unrecorded action. For the shipped\nsingle-action playbooks the two granularities are equivalent in effect;\nthe difference starts with multi-action runs -- where whole-run keying\nwould already be wrong.\n\n**Storage/lookup: a new durable ledger table `playbook_dispatches` in\n`internal/store`, copying `binding_dispatches`'s conventions exactly**\n(durable table, `INSERT OR IGNORE`, sentinel error, rows freed with\ntheir parent, survives daemon restarts -- `internal/store/bindings.go`):\n\n```sql\nCREATE TABLE playbook_dispatches (\n playbook_id TEXT NOT NULL,\n playbook_version TEXT NOT NULL,\n event_id TEXT NOT NULL,\n action_id TEXT NOT NULL,\n dispatched_at TEXT NOT NULL,\n PRIMARY KEY (playbook_id, playbook_version, event_id, action_id)\n);\n```\n\nWritten with the same `INSERT OR IGNORE` + sentinel-error convention\n(`ErrAlreadyDispatched`, `bindings.go`), so a duplicate is a no-op\nwrite rather than a constraint error, and the caller matches it with\n`errors.Is` the way the binding dispatch loop already does\n(`internal/daemon/daemon.go` -- \"another cycle already won this\nrace,\" not an error to surface). The record is consumed by the playbook\ncoordinator at dispatch time through a domain-side interface implemented\nby `internal/store` -- the same shape as `BindingDispatcher`\n(`internal/store/interface.go`) -- and the table lands in the\nstore's schema application (`internal/store/store.go`). No change\nto `internal/eventbus`; no per-message dedup state on the client.\n\n**Lifetime: no time-based expiry.** Rows live as long as their playbook\nexists; when a playbook is removed from the configured directories, the\ncoordinator deletes its rows -- mirroring binding-dispatches-rows-deleted-\nwith-binding (`bindings.go`, same transaction as the delete). The\nat-most-once guarantee must not silently decay, which is exactly the\nproperty that disqualifies `DedupWindow` above; table growth is bounded\nby distinct (playbook, event, action) pairs over the playbook's lifetime\nand is accepted, the same tradeoff `binding_dispatches` already makes.\n\n**Redelivery behavior: skip the recorded action and continue the run.**\nA detected duplicate is not re-invoked, is not reported to the caller as\nan error, and is logged at debug level rather than as a warning -- this\nis the expected, correct behavior of at-least-once delivery, not an\nanomaly (same treatment as `ErrAlreadyDispatched` at\n`daemon.go`).\nThe run continues with the next action (or ends, for a single-action\nplaybook whose one action is recorded), which is what makes a redelivery\na resume rather than a dead restart.\n\n**Key derivation: a fixed composition of four structural identities,\ncomputed by the coordinator -- not a CEL expression.** The key is the\n`(playbook_id, playbook_version, event_id, action_id)` tuple above, where:\n\n- `playbook_id` is `Playbook.ID` -- the playbook file's path relative to\n its configured directory root, slash-normalized, unique within the load\n composition by construction (`internal/domain/eda/playbook`).\n- `playbook_version` is `Playbook.Version` -- a SHA-256 of the loaded\n file, recomputed on every load (`internal/domain/eda/playbook`),\n mirroring `Binding.Version`'s provenance-pinning purpose: the dispatch\n is pinned to the exact definition that was active when it fired. A\n changed playbook definition is a different key and may fire again for\n the same event -- deliberate, not a bug.\n- `event_id` is `DispatchInput.TaskID` (`playbook.go`), which\n carries the `TaskEnvelope.IdempotencyKey()` value\n (`\"archie:\" + owner/repo/number`,\n `internal/domain/workintake/envelope.go`). It is chosen over\n `store.Task.ID` because the trigger vocabulary is issue-level\n workintake label/kind, dispatched at discovery time (pollNATS, forge\n webhook receiver), where no task row exists yet (`playbook.go`\n comment; `playbook_test.go`: \"the value available at the\n discovery/dispatch point... NOT a store.Task.ID int64\"). Keying on the\n issue identity rather than the delivery source is the load-bearing\n property the existing contract already established for `TaskEnvelope`\n (`event-sources-and-reactions.md` question 4: a webhook-sourced\n envelope for the same issue produces the *same* idempotency key as a\n poll-discovered one; `internal/forge/webhook/receiver.go` --\n \"Idempotency is not this package's job\"); this resolution inherits that\n property rather than re-deciding it. Consequence, stated explicitly:\n same-issue re-discoveries (a re-label that re-triggers the same kind)\n are the *same event* by design -- the workintake trigger vocabulary is\n issue-granular, exactly as `TaskEnvelope`'s own key is. A future\n capture/binding-originated trigger type keys on `captured_events.id`\n instead; derivation is per-trigger-type, not one formula for all\n triggers (rule carried over unchanged from the draft).\n- `action_id` is the action's declared `id` -- J1's named ids, \"a\n required field on every action that later actions reference\", unique\n and stable-identifier-shaped, validated at load (the CEL resolution's\n J1 and its context table above in this document). For an action declaring no `id`\n (permitted by J1 while unreferenced), the key uses the action's 1-based\n position in the playbook: deterministic and stable for the\n single-action playbooks, and a later reorder is a file edit whose\n content hash changes `playbook_version` with it, so a positional\n fallback cannot silently collide with a prior definition.\n\n**No CEL in the key -- stated explicitly because the CEL resolution\nflagged this as gap 2's \"first real exercise\"** (\"CEL's read-only result\naccess makes gap 2's keying derivation readable\" -- the CEL resolution's\nclosing note). Readability\nis not the binding constraint; **determinism under redelivery is**, and\nCEL key expressions lose three ways. First, they are not load-checkable:\n`event` stays `dyn` by the resolved J4 (schema-by-example), so\na key expression over event fields cannot be type-checked at load,\nviolating the document's reject-at-load philosophy. Second, they can\nfail at dispatch (a missing field is a CEL error), and a key that cannot\nbe computed is an unfireable state -- neither \"skip\" nor \"fire\" is safe\nwithout losing or duplicating work. Third, the contract needs one\nderivation, and two authors writing different key expressions would\nfragment the ledger. The key is therefore structural by construction:\nevery component above is an immutable identity of the run's inputs, so\ntwo deliveries of the same event derive the same key with no evaluation.\nThe exercise's actual outcome is the derivation rule above; CEL remains\na *condition* surface (`when`), never a key surface. An action kind that\ngenuinely needs finer granularity than (action, event), such as \"per\ncomment, not per issue\", arrives with its own trigger type and its own\nevent identity, per the per-trigger-type rule.\n\n**Record-before-invoke -- one deliberate divergence from\n`binding_dispatches`, justified by the harm profile.** The binding\ndispatch loop enqueues the task first and records the ledger row after\n(`daemon.go`), accepting that racing cycles may both enqueue\n(\"the duplicate task remains queued... delete-on-races is its own can of\nworms\"); the selection query excludes already-dispatched captures\n(`bindings.go`), but the race window remains. That ordering is\ncorrect there because the duplicate artifact is an internal task row --\nrecoverable, low-harm. The playbook ledger's ordering is reversed on\npurpose: **the row is written and committed before the action's side\neffect is invoked** -- the `INSERT OR IGNORE` is the atomic gate, and\nthe lost-write loser skips without invoking. The cost is stated, not\nhidden: a crash between the row commit and the invoke loses that one\naction's side effect, repairable by hand -- the reverse direction of the\nsame window that would otherwise duplicate a posted comment or\nre-bill an image generation. For side effects whose duplication is\nexternally visible and not revocable, at-most-once is the required\ndirection; the binding system's at-least-once-tolerant ordering is not\nthe precedent for this ledger.\n\n**Gap-1 interplay, stated to keep the two mechanisms separate:** the\nledger answers one question per action -- has this action already fired\nfor this event? It does not answer whether the run continues, stops, or\nis re-attempted; that is gap 1's coordinator concern (a run-outcome\nrecord; gap 1 explicitly leaves \"what leave it alone for restart means\nfor a playbook run\" to whoever builds the coordinator). One consequence\nof record-before: an action that FAILED (gap 1's real-action-error case)\nhas a ledger row, so a later redelivery of the same event skips it --\nconsistent with gap 1's \"no automatic retry\" resolve (a corrective\naction is a *new* playbook/action, per gap 1's rollback paragraph). A\nfailed action is never re-attempted by the bus; if a run must be\nrecovered, an operator edits the playbook -- new version, new key --\nrather than fighting ledger state.\n\n**Relationship to `event-sources-and-reactions.md` -- extension, not\ncontradiction.** Keyed to that document's sections:\n\n1. Its decision 2 commits reactions to at-least-once delivery via\n `PublishUnique`'s \"without duplicate enqueue\" guarantee -- which, with\n the code facts above, is accurate for the *publish* path only:\n `PublishUnique` absorbs duplicate publishes of the same event (poll\n vs. webhook), which is what that decision was actually about. This\n document's own problem statement, taken literally, overstates it\n (\"keys absorb redelivery without duplicate side effects\") -- that\n overstatement is exactly what this gap exists to correct. The\n existing contract never claimed consumer redeliveries of\n side-effecting reactions were absorbed, so nothing it says contradicts\n a consumer-side ledger; this resolution pins down the boundary the\n sentence left implicit (publish-path dedup vs. consumer-path dedup)\n rather than changing either.\n2. Its decision 3 is untouched: the ledger is the run's own bookkeeping\n about its own actions, suppressing a re-fire and nothing else.\n3. Its decision 4 (no generic `Source` interface; `TaskEnvelope` is the\n typed contract, keyed on issue identity, not delivery source): the\n event half of the playbook key IS that same identity\n (`TaskEnvelope.IdempotencyKey()`), and the per-trigger-type rule\n means any future trigger type brings its own identity rather than a\n new generic one -- the playbook ledger introduces no second identity\n vocabulary and no untyped payload. The only new type in this\n resolution is the ledger's own value object, consumed through a\n narrow interface like `BindingDispatcher` -- the same shape the\n document's \"typed families\" rule (decision 1, plugin engine rule)\n already blesses.\n\n**Scope: this is the `channel`/`forge`/`module` decision, not a\n`workflow`-position change.** Workflow-kind actions keep their existing\nkeying claim unchanged -- the `TaskEnvelope` path that this document\nalready inherits for free (`daemon.go`, `PublishTask` publishes\nwith `task.IdempotencyKey()`), with its cross-poll/webhook property\nuntouched. The ledger gate applies to the side-effecting positions\n(channel, forge, module), which is why the closing statement below\nstill holds: the first slice (Module + loader + workflow-kind dispatch)\nwas buildable without this resolution, and channel/forge/multi-action\nplaybooks were not. A coordinator MAY implement one position-uniform\ngate as a simplification; what this resolution requires is only that\nthe side-effecting positions are gated.\n\n## Resolved questions\n\n1. **Data-flow/condition syntax.** Does this project want a small existing\n Go expression evaluator (there may be one already in the codebase worth\n checking, e.g. anything backing gate conditions), or a bespoke minimal\n grammar?\n\n **Resolved** (`archie-core-t2db.14`). CEL (`cel.dev/cel-go`, pinned) is\n the one mechanism for both an action's `when` condition and its `args`\n values. The trust boundary, evaluation context, load-time validation and\n the five judgment calls are in\n `docs/prds/playbook-expression-syntax.md`.\n\n2. **Playbook directory config field.** Where this lives in\n `internal/config` / `configuration.Document`, and whether it's one\n directory or a list (mirroring `SkillsDir`'s shape) -- follow existing\n config precedent, not invented fresh.\n\n **Resolved** (`archie-core-t2db.11`). The directory\n shape landed as `playbook_dirs` (a LIST of directories of\n `*.yaml`/`*.yml` binding files), loaded at startup as an additional\n input to the two single-file fields, which remain supported unchanged.\n A single `playbook_dir` would contradict\n supporting multiple independently-maintained playbook sources, which\n the Dedup section already assumes ('independently-sourced playbook\n directories collide'), so the field is a list. Cross-source collision\n (same key in two directories, or in a single-file field and a\n directory) is reported like an in-directory collision -- nothing is\n arbitrated by source precedence. This is what finally exercises the\n design doc's drop-and-report collision rule, since a single file\n cannot collide with itself. Repo-scoping (tying directories to\n `Config.Repos` owner/name) is explicitly deferred to a separate,\n later decision; org/tenant keying is out of scope (no auth/identity\n model).\n\n Decisions recorded from the label-vocabulary slice:\n - Arbitrary labels bind to registered workflow names via\n `LabelWorkflows` + `LoadLabelWorkflowsYAML`/`SetLabelWorkflows`\n (`internal/domain/workflow/routing.go`).\n - The closed `Kind`/NATS-subject set is untouched; the label map only\n extends binding authority for labels the kind layer does not own.\n - Collision rule: a label already owned by the\n kind set (bug/feature/bootstrap), an empty label, an empty workflow\n name, or a duplicate binding is a reported load failure -- dropped\n and logged, the error returned to the caller. Nothing is silently\n arbitrated by schema/version. This is the \"schema defines the\n accepted message; anything else is the caller's fault\" rule applied\n to the label layer.\n - Precedence in `Route()`: explicit `t.Workflow` → arbitrary-label\n binding → kind binding → triage → implement → default.\n3. **Trust boundary for Module Yaegi code.** The plugin engine rule's\n invariant 6 draws a line between operator-trusted in-process code and\n repository-supplied code that must run in a container.\n\n **Resolved** in `docs/prds/module-position.md`, \"Trust boundary\": a\n Module is operator-installed, in-process and daemon-privileged, the same\n tier as `PluginDir` and `SecretEngineDir`, never repository-supplied\n task code.\n4. **First implementation slice.** Recommend: Module position + the\n playbook loader + trigger-to-workflow dispatch only (subsuming the existing\n label routing) first, proving the schema-gen -\u003e Yaegi -\u003e playbook path\n end to end on the smallest useful case, before adding Channel/Forge\n action kinds or the linter/LSP.\n\n **Resolved** (`archie-core-t2db.13`, `.14`, `.15`). The Module position\n (registry + log kind, t2db.13), the CEL expression environment\n (t2db.14), and the playbook document + event coordinator with\n single-action workflow-kind dispatch (t2db.15) are shipped.\n\n **Resolved** (`archie-core-t2db.23`): the coordinator selects a real\n task's workflow. The wiring point is the daemon's definition pin\n (`Daemon.resolveWorkflowID`, consumed by `pinWorkflowFromCollection` in\n `internal/daemon`), not `pollNATS`/`publishTask`. Routing belongs to the\n pin because a workflow definition is a\n database row pinned before dispatch, which makes the pin the one place a\n task's workflow is chosen. Precedence there is explicit `Task.Workflow`\n (the waiting_human -\u003e approved requeue) → matching playbook →\n `workflow.ResolveWorkflowID`'s label binding → kind binding → triage →\n implement. A playbook naming an undefined workflow is a reported failure\n that parks the task rather than a silent fall-through to the binding's\n choice.\n\n `Store.Dispatch` returns the selected workflow *name* and the matching\n playbook's id and version, not a compiled `workflow.Workflow`: the\n production caller decides against the active definition collection, and\n the id and version are the first two components of the gap-2 idempotency\n key. The playbook package does not import `internal/domain/workflow`.\n\n A `when` condition at this point reads what the intake knows about the\n originating issue: `kind`, `labels`, `owner`, `repo`, `number`, `title`,\n `body`, `source`, built by `playbookInput` in `internal/daemon`.\n\n The trigger shape reuses the existing workintake kind/label vocabulary.\n The single-action hard boundary is superseded by\n `docs/prds/multi-action-playbooks.md` (two-shape rule); Module, Channel,\n and Forge positions still have no production dispatch path.\n## Standing constraint\n\nThis may be commercialized, and is to be discernible from other open-source\nevent-driven-automation tooling. No design decision follows from that yet;\nit bears on licensing or feature-gating questions about playbook authoring\nand the LSP when they arise.\n\n## Packages this touches (first slice)\n\n- New: a schema-generation package/convention (`go:generate` source +\n emitted contracts) for the Module position.\n- New: playbook loader + event coordinator (name/location TBD -- likely\n `internal/domain/eda` or similar, following the domain-layer rule: no\n infrastructure imports, no untyped hooks).\n- `internal/domain/workflow`: dispatch target for `workflow`-kind actions,\n unchanged in its own contracts.\n- `internal/domain/workintake`: `labelKinds`/routing becomes one playbook\n among others once the loader subsumes it; existing Go fallback stays for\n backward compatibility until playbooks fully replace it (mirrors the\n `dynamic-workflow-triage.md` precedent of a two-line fallback change, not\n a rip-and-replace).\n" + "body": "# EDA playbook engine -- design\n\n**Status:** Draft\n**Date:** 2026-09-03\n**Beads epic:** `archie-core-t2db`\n\n## Problem\n\n\"Which workflow runs for which trigger\" is hardcoded Go: a literal\nlabel vocabulary (`workintake.labelKinds`) and a literal label-\u003eworkflow map\n(`workflow/routing.go`), extended only by adding a Go case or shipping a\nskill whose `metadata.archie.workflow` overrides one *named* built-in\nworkflow. Extending the system to a new trigger, a new action, or a new\nevent source means a feature branch, a code change in a hardcoded\npath, and new Go tests -- for every addition. That is the barrier this\ndocument exists to remove.\n\nThe goal: triggers, the actions they run, and the\nconditions that connect them should be **drop-in and load**, not **drop-in\nand touch application code**. Yaegi-evaluated Go is the unit of imperative\n\"do a thing\" logic; YAML playbooks are pure orchestration data -- no logic,\njust \"on this event, with this data, call these actions, in this order,\nunder these conditions.\" The daemon's event coordinator doesn't know what a\n`bugfix` or `image-gen` action does, it resolves the name against a\nregistry and pipes typed data through.\n\nThis is explicitly **not** scoped to code-generation workflows. A playbook\naction can be any registered capability -- run a codegen workflow, send a\nmessage on a channel, open something on a forge, generate an image -- so\nlong as it's implemented behind one of the typed engine families below.\n\n## Relationship to existing decisions\n\n- `docs/prds/event-sources-and-reactions.md` ruled\n that reactions in the two epics it covers are **producer-only** (no veto,\n no mutation of in-flight work) and explicitly rejected a single generic\n `Source` interface as \"the untyped-hook shape the plugin engine rule\n exists to prevent.\" **This document extends that scope**: playbooks can\n dispatch to multiple typed action kinds, not just \"produce a\n `TaskEnvelope`.\" It does not relitigate producer-only-ness (still no\n veto/mutation of in-flight work here) and it does not introduce one\n generic interface -- see \"Typed positions, not one generic hook\" below,\n which is how it stays compliant with the plugin engine rule instead of\n reopening that document's #4.\n- `docs/architecture/plugins-and-extensions.md#plugin-engine-rule-strict`\n governs every extension point\n introduced here. Every position below is a typed domain contract with an\n owning registry, not a generic callback map. This is a hard constraint,\n confirmed against the existing rule text before writing this doc.\n- Yaegi is already a proven in-process extension mechanism\n (`internal/plugin`, `internal/domain/workflow/wfeval/yaegi.go`,\n `internal/gate/gateeval/yaegi.go`, `internal/skill/plugin.go`) -- this\n design generalizes its existing role, it does not introduce a new\n execution mechanism.\n- `internal/domain/workflow/skillbuild.BuildRegistry` already composes a\n `workflow.Registry` from a skills catalog, plugin-defined overriding\n built-in. The playbook loader described here supersedes label routing\n specifically; it does not replace `skillbuild`'s workflow-composition role,\n it becomes an additional input the daemon composes at startup.\n- Webhook event intake is a separate design. This\n document defines the router each webhook-sourced event should be able to\n reach; it does not redesign webhook receipt itself.\n\n## Shape: four logic positions, each a typed engine family\n\nEvery entrypoint kind (mechanism, action, loop, schedule, route, trigger) is itself a schema emitter: defining a new kind produces the\ngenerated contract downstream tooling consumes. The kinds slot into a fixed\nhierarchy of logic positions:\n\n```\nWorkflow -\u003e Module -\u003e Channel (telegram/whatsapp/email/...)\n -\u003e Forge (GitHub/Gitea/GitLab/...)\n```\n\nEach position is its own typed engine family per the plugin engine rule:\nidentity + capability-specific operations, an owning `Registry`/`Manager`,\nexplicit lifecycle where the implementation owns resources, narrow host\naccess. Concretely:\n\n- **Workflow** -- already exists (`internal/domain/workflow`). A playbook\n action of kind `workflow` invokes a registered `workflow.Registry` entry\n by name, unchanged from the existing execution path.\n- **Module** -- the new general-purpose action position. A `Module` is a\n Yaegi-evaluated implementation of a generated interface (args in, typed\n result out) for one declared action kind (e.g. `image-gen`, `notify`,\n `open-pr-comment`). This is the position that makes actions \"not\n restricted to code-generation.\" Full design (schema-per-kind, the\n registry shape, the trust boundary, the recommended first slice):\n `docs/prds/module-position.md`.\n- **Channel** -- already exists as `channels.Channel` (telegram, email,\n webhook). A playbook action of kind `channel` sends through a named,\n already-registered channel. No new mechanism; this document just makes it\n addressable from a playbook.\n- **Forge** -- forge clients (GitHub, Gitea, GitLab) become an addressable\n action position the same way Channel is, so a playbook can say \"comment\n on this forge issue\" without that being a special case wired only into\n the poller.\n\n**Typed positions, not one generic hook.** There is no single `Action`\ninterface with an `any` payload. Each position keeps its own typed\ncontract; the playbook YAML names a position and an implementation, and the\nevent coordinator resolves that pair against the matching typed registry.\nThis is what keeps the design compliant with the plugin engine rule and\nwith `event-sources-and-reactions.md`'s rejection of a generic `Source`/\nhook interface -- the generality lives in the YAML data model (which\npositions exist, which named implementations are registered), not in a\nruntime `any`-typed dispatch path.\n\n## `go:generate`-driven schemas, from day one\n\nSchemas are not a future nicety bolted onto an ad hoc\nYAML shape. They are the struct+interface contracts for each logic\nposition, and they drive code generation from the start.\n\n- Each logic position (Workflow/Module/Channel/Forge) has a schema source\n (Go-annotated types, following the existing pattern elsewhere in the\n repo) from which `go:generate` emits the interface and struct scaffolding\n an implementation must satisfy.\n- A Yaegi module implements the generated interface for its position. The\n generated contract is what makes an implementation valid, not hand-review.\n- The schema is also what the linter and LSP (below) validate a playbook\n YAML against -- one source of truth for codegen, startup loading, and\n authoring-time tooling, so the three can't drift from each other.\n- Schema versioning is explicit per position, independent of a playbook's\n own workflow version (see collision handling below).\n\nThe generator itself is not designed here. The constraint is\nschema-first, `go:generate`-driven, per logic position.\n\n## Playbook YAML: pure orchestration data\n\nA playbook declares no embedded logic -- only:\n\n- **trigger**: the event predicate (e.g. forge issue assigned + labelled\n `bugfix`, a schedule tick, a webhook payload shape).\n- **actions**: an ordered list, each naming a logic position and a\n registered implementation, with typed argument data and optional\n conditions.\n- **data flow between actions**: results/facts from one action available to\n later actions in the same playbook run -- exact interpolation syntax is\n an open point, not decided here (see Open questions).\n\nPlaybooks live in a configured directory (repo/directory-scoped, per Sam's\n\"define repos, directories, playbooks\" framing) and are loaded at archied\nstartup, alongside the existing skills-catalog composition.\n\n## Dedup: detect and log, no arbitration mechanism\n\nNo runtime auto-resolution rule. An automatic \"pick a winner\" algorithm\n(e.g. by version) is itself a mechanism that can fail and need debugging\nwhen a collision happens -- the wrong kind of complexity to introduce for\na case that should just be visible and fixed by a human. Instead:\n\n- **At daemon startup:** if independently-sourced playbook directories\n declare a collision on the same trigger, that is a definition failure.\n The colliding definitions are dropped (not loaded) and the failure is\n logged plainly (which sources, which trigger) and reported back to the\n caller that requested the load -- not swallowed as a background log line\n only. The schema is the definition; a schema conflict is a reported\n failure, not something the daemon silently arbitrates.\n- **Linter**: a CI/dev-time tool, reading the same generated schemas, that\n flags duplicate or conflicting trigger definitions *within* a single\n maintained tree before merge.\n\n **Resolved** (`archie-core-t2db.12`). A standalone\n gopls-shaped binary (`cmd/archie-playbooks`, lint subcommand) validates\n playbook directories against the exact loaders the daemon uses\n (`LoadPlaybookDirs`/`LoadKindWorkflowsYAML`/`LoadLabelWorkflowsYAML`),\n exiting non-zero on any collision / malformed file / invalid binding.\n Findings are file-granular, not line-granular: the loader decodes with\n `yaml.Unmarshal` into a plain map, which discards line numbers. A\n compiler-style file:line diagnostic needs a `yaml.Node` decoding\n upgrade, tracked separately -- the linter agrees with runtime\n validation by construction, which is the load-bearing property.\n Discoverable via `task lint:playbooks` or direct `go run\n ./cmd/archie-playbooks lint -dir ...`. The LSP/serve mode is a later\n entrypoint of the SAME binary, per the shared-package decision above.\n- **LSP**: same schema source again, feeding author-time hover/completion/\n inline errors while someone edits a playbook YAML in an editor. Kept\n explicitly in scope per Sam (not deferred), because it's the same schema\n artifact as the linter's, just a different consumer.\n\nThe startup log-and-refuse path exists for what the linter/LSP structurally\ncannot see -- sources composed together only at daemon startup, never\nchecked against each other by either tool. If the linter and LSP do the job\nthey're built for, a real collision reaching that startup path should be\nrare in practice, not a routine occurrence the design leans on.\n\n**Shared package, thin consumers -- not a bundled-vs-standalone choice.**\nThe validation logic (`LoadKindWorkflowsYAML`, `LoadLabelWorkflowsYAML`, the\ndirectory-merge from t2db.11) lives in `internal/domain/workflow`, a plain\nGo package with no knowledge of which binary calls it. This makes \"should\nthe linter/LSP be bundled into archied or a separate tool\" a non-question:\n- `archied` already imports the package directly and validates in-process\n at startup -- no subprocess, no external tool call.\n- An agent that needs to validate a playbook in-process can import the same\n package directly, for the same reason.\n- The standalone `cmd/` binary (t2db.12, gopls-shaped: one binary, a lint\n mode now, an LSP/serve mode later) exists only because CI and editors are\n out-of-process consumers by nature -- a CI step needs an exit code, an\n editor speaks LSP over a process boundary, neither can `import` a Go\n package. It is a thin wrapper per `organisation.md`'s `cmd/` rule, not a\n second implementation.\n\nFolding CLI/LSP protocol handling into `archied` itself would grow the\ndaemon into unrelated concerns, against the plugin-engine rule's spirit of\nnarrow capability families.\n\n## Execution-time gaps (both resolved)\n\nEverything above covers *loading* a playbook. Nothing above covers what\nhappens while one *runs*. Two gaps, both load-bearing enough to resolve\nbefore implementation, not defer:\n\n### 1. Mid-playbook action failure\n\n**Resolved** (`archie-core-t2db.18`). The default below\nis a committed decision, not a candidate list; changing it later would\nrequire its own decision. Gap 2 below is resolved by `archie-core-t2db.17`.\n\nA playbook is an ordered action list with data flowing between steps. If\naction 2 of 4 fails at runtime (an image-gen API times out, a forge call\n403s), the run needs a defined outcome. This\ncodebase already answers a structurally identical question for its\nexisting workflow engine, precisely enough to reuse rather than\nre-derive: `workflow.Run` (`internal/domain/workflow/workflow.go`)\ndraws a three-way distinction on a stage failure, and a playbook run\ndraws the same one on an action failure.\n\n**Decision: reuse `Run`'s three-way distinction directly.** The stated\ndefault is **stop-on-first-failure**: a real action error halts the\nplaybook run immediately and is reported, with no continue-to-next-action\nand no automatic retry or compensating follow-up. The three points below\nare the complete decision.\n\n1. **A real action error** (the action's own logic failed -- an API\n error, a validation failure inside the action) stops the run\n immediately. No later actions execute. The failure is reported the\n same way a load collision is reported -- dropped, logged plainly\n (which action and why), and returned to the caller, never swallowed\n as a background log line only (the Dedup section's drop-and-report\n rule). `Run` is the mechanical\n precedent for the stop itself: a stage error sets the park reason and\n returns without running further stages (`workflow.go`). This\n is stop-on-first-failure, matching the document's existing bias\n against clever automatic recovery, and it is now a stated decision\n rather than a nobody-chose default.\n2. **Interruption** (the daemon is shutting down mid-dispatch, `ctx.Err()\n != nil`) is explicitly **not** a failure. `Run` already treats this\n case specially -- \"Daemon shutdown is not a workflow failure. Leave\n the task running ...; parking here would publish a false failure and\n require manual intervention\" (`workflow.go`) -- and the same\n reasoning applies to a playbook run: an in-flight dispatch interrupted\n by restart must not be recorded as a broken playbook. What \"leave it\n alone for restart\" means for a playbook run (there is no `store.Task`\n row a playbook run inherently owns the way a workflow stage does) is\n an open implementation detail for whoever builds the coordinator, not\n decided here -- but the *semantic* (interruption ≠ failure) is decided.\n3. **A run that produces no outcome at all** (every action ran without\n error, but nothing was assigned to happen) is itself an error\n condition, matching `Run`'s own \"a workflow must end with an explicit\n outcome; not doing so is a definition bug, which still must not vanish\n silently\" (`workflow.go`). A playbook with actions that all ran\n but never reached a coherent terminal state is a definition bug in the\n playbook, reported as such.\n\n**Rollback:** explicitly **not attempted**, for exactly the reason the\ngap's own description named -- a posted forge comment is not revocable.\nThis is not a limitation to work around later; it is the correct\nsemantic. An operator who needs a corrective action after a failure\nwrites a *new* playbook/action for that, they do not get automatic\nundo.\n\n**This stays inside the producer-only rule.** Stop-on-first-failure is the\nrun deciding its own terminal state from its own action's error. Nothing\nalready in flight is blocked or altered; the run only declines to start\nactions that had not started yet, and it records the failure on its own\nunit the way `Run` sets a park reason and transitions the task to\n`StatusParked`. A future compensating action (\"if the notify action\nfailed, also log an incident\") would be producing new work, which the rule\nalready permits. No syntax for it is designed here.\n\n**J3 relationship, stated explicitly:** J3 (a CEL `when` condition\nerroring at eval time → treated as false → skip that action → run\ncontinues) is a **different case from an action failure** and is not\nrevisited by this resolution. A condition error means \"we couldn't\ndetermine whether to run this action,\" which this document already\nresolved as \"then don't, and move on.\" An action failure means \"we tried\nto run this action and it broke,\" which stops the run per point 1 above.\nThe two must not be conflated: a coordinator implementation should be\nable to point at one `when`-evaluation code path and one\naction-invocation code path and show they use different outcomes.\n\n### 2. Idempotency at execution time\n\n**Resolved** (`archie-core-t2db.17`, a\ndesign-investigation ticket: resolve, do not implement). The decision\nbelow is the answer; the keying scheme itself is separate follow-up work\nand is intentionally not built here. It **extends**\n`event-sources-and-reactions.md`'s existing contract -- it does not\ncontradict it (relationship argued at the end).\n\n**Problem (unchanged):** `event-sources-and-reactions.md` decided\nreactions ride `internal/eventbus`'s at-least-once delivery specifically\nso `PublishUnique`/idempotency keys absorb redelivery without duplicate\nside effects. This document inherits that for free only for\n`workflow`-kind actions, because `TaskEnvelope` already carries an\nidempotency key. It said nothing for `channel`, `forge`, or `module`\nactions with real side effects -- a posted comment, a sent message, a\nbilled image generation.\n\nOne correction to the problem statement, checked against the code rather\nthan assumed, because it changes what the answer must be:\n**`PublishUnique`'s dedup is not a consumer-redelivery guard at all.**\n`PublishUnique` only sets the JetStream `Nats-Msg-Id` header on the\noutgoing message (`internal/infrastructure/eventbus/nats/publisher.go`;\nkey constant `message.go`); JetStream suppresses *republished*\nmessages carrying a repeated key inside `Config.DedupWindow`\n(`client.go`, `Duplicates: cfg.DedupWindow`), default 2 minutes\n(`config.go`, `DefaultDedupWindow = 2 * time.Minute`). A consumer\nredelivery is the same *stored* message re-delivered on Nak or\nacknowledge-timeout (`internal/eventbus/eventbus.go`: \"a handler\nreturning an error causes redelivery\") -- not a republish; its\n`Nats-Msg-Id` is never re-firewalled, and even if it were, two minutes is\nnot an at-most-once guarantee for a side effect that must never repeat.\n**Conclusion: side-effecting action kinds need a consumer-side, durable\ndedup record; the bus cannot supply it.** That is precisely the situation\n`binding_dispatches` was built for -- so the decision below reuses its\nmechanism rather than designing a new one.\n\n#### Decision\n\n**Granularity: per-action-per-event -- one dedup record per\n(action, event) pair, not per playbook run.** Whole-run keying, the\nobvious alternative, has two crash windows: the run row is written when the run starts, so a crash mid-run\nleaves the run \"started\" and a redelivery skips *every remaining\naction* (work silently lost); a crash before the row write re-runs\n*every already-fired action* (duplicate side effects -- the exact harm\nthis gap exists to prevent). Per-action keys collapse both into one\nwindow and one behavior: an action's record is written immediately\nbefore that action's side effect is invoked, so an action fires iff its\nown record is absent, no fired action ever re-fires, and a redelivered\nevent resumes at the first unrecorded action. For the shipped\nsingle-action playbooks the two granularities are equivalent in effect;\nthe difference starts with multi-action runs -- where whole-run keying\nwould already be wrong.\n\n**Storage/lookup: a new durable ledger table `playbook_dispatches` in\n`internal/store`, copying `binding_dispatches`'s conventions exactly**\n(durable table, `INSERT OR IGNORE`, sentinel error, rows freed with\ntheir parent, survives daemon restarts -- `internal/store/bindings.go`):\n\n```sql\nCREATE TABLE playbook_dispatches (\n playbook_id TEXT NOT NULL,\n playbook_version TEXT NOT NULL,\n event_id TEXT NOT NULL,\n action_id TEXT NOT NULL,\n dispatched_at TEXT NOT NULL,\n PRIMARY KEY (playbook_id, playbook_version, event_id, action_id)\n);\n```\n\nWritten with the same `INSERT OR IGNORE` + sentinel-error convention\n(`ErrAlreadyDispatched`, `bindings.go`), so a duplicate is a no-op\nwrite rather than a constraint error, and the caller matches it with\n`errors.Is` the way the binding dispatch loop already does\n(`internal/daemon/daemon.go` -- \"another cycle already won this\nrace,\" not an error to surface). The record is consumed by the playbook\ncoordinator at dispatch time through a domain-side interface implemented\nby `internal/store` -- the same shape as `BindingDispatcher`\n(`internal/store/interface.go`) -- and the table lands in the\nstore's schema application (`internal/store/store.go`). No change\nto `internal/eventbus`; no per-message dedup state on the client.\n\n**Lifetime: no time-based expiry.** Rows live as long as their playbook\nexists; when a playbook is removed from the configured directories, the\ncoordinator deletes its rows -- mirroring binding-dispatches-rows-deleted-\nwith-binding (`bindings.go`, same transaction as the delete). The\nat-most-once guarantee must not silently decay, which is exactly the\nproperty that disqualifies `DedupWindow` above; table growth is bounded\nby distinct (playbook, event, action) pairs over the playbook's lifetime\nand is accepted, the same tradeoff `binding_dispatches` already makes.\n\n**Redelivery behavior: skip the recorded action and continue the run.**\nA detected duplicate is not re-invoked, is not reported to the caller as\nan error, and is logged at debug level rather than as a warning -- this\nis the expected, correct behavior of at-least-once delivery, not an\nanomaly (same treatment as `ErrAlreadyDispatched` at\n`daemon.go`).\nThe run continues with the next action (or ends, for a single-action\nplaybook whose one action is recorded), which is what makes a redelivery\na resume rather than a dead restart.\n\n**Key derivation: a fixed composition of four structural identities,\ncomputed by the coordinator -- not a CEL expression.** The key is the\n`(playbook_id, playbook_version, event_id, action_id)` tuple above, where:\n\n- `playbook_id` is `Playbook.ID` -- the playbook file's path relative to\n its configured directory root, slash-normalized, unique within the load\n composition by construction (`internal/domain/eda/playbook`).\n- `playbook_version` is `Playbook.Version` -- a SHA-256 of the loaded\n file, recomputed on every load (`internal/domain/eda/playbook`),\n mirroring `Binding.Version`'s provenance-pinning purpose: the dispatch\n is pinned to the exact definition that was active when it fired. A\n changed playbook definition is a different key and may fire again for\n the same event -- deliberate, not a bug.\n- `event_id` is `DispatchInput.TaskID` (`playbook.go`), which\n carries the `TaskEnvelope.IdempotencyKey()` value\n (`\"archie:\" + owner/repo/number`,\n `internal/domain/workintake/envelope.go`). It is chosen over\n `store.Task.ID` because the trigger vocabulary is issue-level\n workintake label/kind, dispatched at discovery time (pollNATS, forge\n webhook receiver), where no task row exists yet (`playbook.go`\n comment; `playbook_test.go`: \"the value available at the\n discovery/dispatch point... NOT a store.Task.ID int64\"). Keying on the\n issue identity rather than the delivery source is the load-bearing\n property the existing contract already established for `TaskEnvelope`\n (`event-sources-and-reactions.md` question 4: a webhook-sourced\n envelope for the same issue produces the *same* idempotency key as a\n poll-discovered one; `internal/forge/webhook/receiver.go` --\n \"Idempotency is not this package's job\"); this resolution inherits that\n property rather than re-deciding it. Consequence, stated explicitly:\n same-issue re-discoveries (a re-label that re-triggers the same kind)\n are the *same event* by design -- the workintake trigger vocabulary is\n issue-granular, exactly as `TaskEnvelope`'s own key is. A future\n capture/binding-originated trigger type keys on `captured_events.id`\n instead; derivation is per-trigger-type, not one formula for all\n triggers (rule carried over unchanged from the draft).\n- `action_id` is the action's declared `id` -- J1's named ids, \"a\n required field on every action that later actions reference\", unique\n and stable-identifier-shaped, validated at load (the CEL resolution's\n J1 and its context table above in this document). For an action declaring no `id`\n (permitted by J1 while unreferenced), the key uses the action's 1-based\n position in the playbook: deterministic and stable for the\n single-action playbooks, and a later reorder is a file edit whose\n content hash changes `playbook_version` with it, so a positional\n fallback cannot silently collide with a prior definition.\n\n**No CEL in the key -- stated explicitly because the CEL resolution\nflagged this as gap 2's \"first real exercise\"** (\"CEL's read-only result\naccess makes gap 2's keying derivation readable\" -- the CEL resolution's\nclosing note). Readability\nis not the binding constraint; **determinism under redelivery is**, and\nCEL key expressions lose three ways. First, they are not load-checkable:\n`event` stays `dyn` by the resolved J4 (schema-by-example), so\na key expression over event fields cannot be type-checked at load,\nviolating the document's reject-at-load philosophy. Second, they can\nfail at dispatch (a missing field is a CEL error), and a key that cannot\nbe computed is an unfireable state -- neither \"skip\" nor \"fire\" is safe\nwithout losing or duplicating work. Third, the contract needs one\nderivation, and two authors writing different key expressions would\nfragment the ledger. The key is therefore structural by construction:\nevery component above is an immutable identity of the run's inputs, so\ntwo deliveries of the same event derive the same key with no evaluation.\nThe exercise's actual outcome is the derivation rule above; CEL remains\na *condition* surface (`when`), never a key surface. An action kind that\ngenuinely needs finer granularity than (action, event), such as \"per\ncomment, not per issue\", arrives with its own trigger type and its own\nevent identity, per the per-trigger-type rule.\n\n**Record-before-invoke -- one deliberate divergence from\n`binding_dispatches`, justified by the harm profile.** The binding\ndispatch loop enqueues the task first and records the ledger row after\n(`daemon.go`), accepting that racing cycles may both enqueue\n(\"the duplicate task remains queued... delete-on-races is its own can of\nworms\"); the selection query excludes already-dispatched captures\n(`bindings.go`), but the race window remains. That ordering is\ncorrect there because the duplicate artifact is an internal task row --\nrecoverable, low-harm. The playbook ledger's ordering is reversed on\npurpose: **the row is written and committed before the action's side\neffect is invoked** -- the `INSERT OR IGNORE` is the atomic gate, and\nthe lost-write loser skips without invoking. The cost is stated, not\nhidden: a crash between the row commit and the invoke loses that one\naction's side effect, repairable by hand -- the reverse direction of the\nsame window that would otherwise duplicate a posted comment or\nre-bill an image generation. For side effects whose duplication is\nexternally visible and not revocable, at-most-once is the required\ndirection; the binding system's at-least-once-tolerant ordering is not\nthe precedent for this ledger.\n\n**Gap-1 interplay, stated to keep the two mechanisms separate:** the\nledger answers one question per action -- has this action already fired\nfor this event? It does not answer whether the run continues, stops, or\nis re-attempted; that is gap 1's coordinator concern (a run-outcome\nrecord; gap 1 explicitly leaves \"what leave it alone for restart means\nfor a playbook run\" to whoever builds the coordinator). One consequence\nof record-before: an action that FAILED (gap 1's real-action-error case)\nhas a ledger row, so a later redelivery of the same event skips it --\nconsistent with gap 1's \"no automatic retry\" resolve (a corrective\naction is a *new* playbook/action, per gap 1's rollback paragraph). A\nfailed action is never re-attempted by the bus; if a run must be\nrecovered, an operator edits the playbook -- new version, new key --\nrather than fighting ledger state.\n\n**Relationship to `event-sources-and-reactions.md` -- extension, not\ncontradiction.** Keyed to that document's sections:\n\n1. Its decision 2 commits reactions to at-least-once delivery via\n `PublishUnique`'s \"without duplicate enqueue\" guarantee -- which, with\n the code facts above, is accurate for the *publish* path only:\n `PublishUnique` absorbs duplicate publishes of the same event (poll\n vs. webhook), which is what that decision was actually about. This\n document's own problem statement, taken literally, overstates it\n (\"keys absorb redelivery without duplicate side effects\") -- that\n overstatement is exactly what this gap exists to correct. The\n existing contract never claimed consumer redeliveries of\n side-effecting reactions were absorbed, so nothing it says contradicts\n a consumer-side ledger; this resolution pins down the boundary the\n sentence left implicit (publish-path dedup vs. consumer-path dedup)\n rather than changing either.\n2. Its decision 3 is untouched: the ledger is the run's own bookkeeping\n about its own actions, suppressing a re-fire and nothing else.\n3. Its decision 4 (no generic `Source` interface; `TaskEnvelope` is the\n typed contract, keyed on issue identity, not delivery source): the\n event half of the playbook key IS that same identity\n (`TaskEnvelope.IdempotencyKey()`), and the per-trigger-type rule\n means any future trigger type brings its own identity rather than a\n new generic one -- the playbook ledger introduces no second identity\n vocabulary and no untyped payload. The only new type in this\n resolution is the ledger's own value object, consumed through a\n narrow interface like `BindingDispatcher` -- the same shape the\n document's \"typed families\" rule (decision 1, plugin engine rule)\n already blesses.\n\n**Scope: this is the `channel`/`forge`/`module` decision, not a\n`workflow`-position change.** Workflow-kind actions keep their existing\nkeying claim unchanged -- the `TaskEnvelope` path that this document\nalready inherits for free (`daemon.go`, `PublishTask` publishes\nwith `task.IdempotencyKey()`), with its cross-poll/webhook property\nuntouched. The ledger gate applies to the side-effecting positions\n(channel, forge, module), which is why the closing statement below\nstill holds: the first slice (Module + loader + workflow-kind dispatch)\nwas buildable without this resolution, and channel/forge/multi-action\nplaybooks were not. A coordinator MAY implement one position-uniform\ngate as a simplification; what this resolution requires is only that\nthe side-effecting positions are gated.\n\n## Resolved questions\n\n1. **Data-flow/condition syntax.** Does this project want a small existing\n Go expression evaluator (there may be one already in the codebase worth\n checking, e.g. anything backing gate conditions), or a bespoke minimal\n grammar?\n\n **Resolved** (`archie-core-t2db.14`). CEL (`cel.dev/cel-go`, pinned) is\n the one mechanism for both an action's `when` condition and its `args`\n values. The trust boundary, evaluation context, load-time validation and\n the five judgment calls are in\n `docs/prds/playbook-expression-syntax.md`.\n\n2. **Playbook directory config field.** Where this lives in\n `internal/config` / `configuration.Document`, and whether it's one\n directory or a list (mirroring `SkillsDir`'s shape) -- follow existing\n config precedent, not invented fresh.\n\n **Resolved** (`archie-core-t2db.11`). The directory\n shape landed as `playbook_dirs` (a LIST of directories of\n `*.yaml`/`*.yml` binding files), loaded at startup as an additional\n input to the two single-file fields, which remain supported unchanged.\n A single `playbook_dir` would contradict\n supporting multiple independently-maintained playbook sources, which\n the Dedup section already assumes ('independently-sourced playbook\n directories collide'), so the field is a list. Cross-source collision\n (same key in two directories, or in a single-file field and a\n directory) is reported like an in-directory collision -- nothing is\n arbitrated by source precedence. This is what finally exercises the\n design doc's drop-and-report collision rule, since a single file\n cannot collide with itself. Repo-scoping (tying directories to\n `Config.Repos` owner/name) is explicitly deferred to a separate,\n later decision; org/tenant keying is out of scope (no auth/identity\n model).\n\n Decisions recorded from the label-vocabulary slice:\n - Arbitrary labels bind to registered workflow names via\n `LabelWorkflows` + `LoadLabelWorkflowsYAML`/`SetLabelWorkflows`\n (`internal/domain/workflow/routing.go`).\n - The closed `Kind`/NATS-subject set is untouched; the label map only\n extends binding authority for labels the kind layer does not own.\n - Collision rule: a label already owned by the\n kind set (bug/feature/bootstrap), an empty label, an empty workflow\n name, or a duplicate binding is a reported load failure -- dropped\n and logged, the error returned to the caller. Nothing is silently\n arbitrated by schema/version. This is the \"schema defines the\n accepted message; anything else is the caller's fault\" rule applied\n to the label layer.\n - Precedence in `Route()`: explicit `t.Workflow` → arbitrary-label\n binding → kind binding → triage → implement → default.\n3. **Trust boundary for Module Yaegi code.** The plugin engine rule's\n invariant 6 draws a line between operator-trusted in-process code and\n repository-supplied code that must run in a container.\n\n **Resolved** in `docs/prds/module-position.md`, \"Trust boundary\": a\n Module is operator-installed, in-process and daemon-privileged, the same\n tier as `PluginDir` and `SecretEngineDir`, never repository-supplied\n task code.\n4. **First implementation slice.** Recommend: Module position + the\n playbook loader + trigger-to-workflow dispatch only (subsuming the existing\n label routing) first, proving the schema-gen -\u003e Yaegi -\u003e playbook path\n end to end on the smallest useful case, before adding Channel/Forge\n action kinds or the linter/LSP.\n\n **Resolved** (`archie-core-t2db.13`, `.14`, `.15`). The Module position\n (registry + log kind, t2db.13), the CEL expression environment\n (t2db.14), and the playbook document + event coordinator with\n single-action workflow-kind dispatch (t2db.15) are shipped.\n\n **Resolved** (`archie-core-t2db.23`): the coordinator selects a real\n task's workflow. The wiring point is the daemon's definition pin\n (`Daemon.resolveWorkflowID`, consumed by `pinWorkflowFromCollection` in\n `internal/daemon`), not `pollNATS`/`publishTask`. Routing belongs to the\n pin because a workflow definition is a\n database row pinned before dispatch, which makes the pin the one place a\n task's workflow is chosen. Precedence there is explicit `Task.Workflow`\n (the waiting_human -\u003e approved requeue) → matching playbook →\n `workflow.ResolveWorkflowID`'s label binding → kind binding → triage →\n implement. A playbook naming an undefined workflow is a reported failure\n that parks the task rather than a silent fall-through to the binding's\n choice.\n\n `Store.Dispatch` returns the selected workflow *name* and the matching\n playbook's id and version, not a compiled `workflow.Workflow`: the\n production caller decides against the active definition collection, and\n the id and version are the first two components of the gap-2 idempotency\n key. The playbook package does not import `internal/domain/workflow`.\n\n A `when` condition at this point reads what the intake knows about the\n originating issue: `kind`, `labels`, `owner`, `repo`, `number`, `title`,\n `body`, `source`, built by `playbookInput` in `internal/daemon`.\n\n The trigger shape reuses the existing workintake kind/label vocabulary.\n The single-action hard boundary is superseded by\n `docs/prds/multi-action-playbooks.md` (two-shape rule); Module, Channel,\n and Forge positions still have no production dispatch path.\n## Standing constraint\n\nThis may be commercialized, and is to be discernible from other open-source\nevent-driven-automation tooling. No design decision follows from that yet;\nit bears on licensing or feature-gating questions about playbook authoring\nand the LSP when they arise.\n\n## Packages this touches (first slice)\n\n- New: a schema-generation package/convention (`go:generate` source +\n emitted contracts) for the Module position.\n- New: playbook loader + event coordinator (name/location TBD -- likely\n `internal/domain/eda` or similar, following the domain-layer rule: no\n infrastructure imports, no untyped hooks).\n- `internal/domain/workflow`: dispatch target for `workflow`-kind actions,\n unchanged in its own contracts.\n- `internal/domain/workintake`: `labelKinds`/routing becomes one playbook\n among others once the loader subsumes it; existing Go fallback stays for\n backward compatibility until playbooks fully replace it (mirrors the\n `dynamic-workflow-triage.md` precedent of a two-line fallback change, not\n a rip-and-replace).\n" }, { "url": "/dev/prds/embedded-nats/", @@ -238,7 +238,7 @@ "url": "/dev/prds/event-sources-and-reactions/", "title": "Event sources and reactions -- decision", "section": "PRDs", - "body": "# Event sources and reactions -- decision\n\n**Status:** Approved\n**Date:** 2026-08-22\n**Beads issue:** `archie-core-7d5u.1`, blocks `7d5u.2/.3/.4/.5`\n\nAnswers the four questions in `7d5u.1`'s description. Investigated against\nthe actual tree, not designed from scratch: three mechanisms already in this\ncodebase answer most of this, and the job here is picking the right one for\neach question and saying which packages change.\n\n## 1. In-process typed families, not out-of-process dispatch\n\nArchie has no out-of-process extension mechanism. The plugin engine\nrule (`ARCHITECTURE.md#plugin-engine-rule-strict`, mechanically checked by\n`internal/plugin/architecture_test.go`) keeps every capability family\nin-process behind a narrow typed registrar; NATS RPC exists for\ndaemon\u003c-\u003eagentexec, a different actor with a different trust boundary (task\nexecution, not third-party extension code), not a precedent for generic\nplugin dispatch. Neither epic's near-term payoff -- react to a labelled\nissue, bind a captured webhook to a workflow -- needs extension code in\nanother language or process. Building out-of-process dispatch now would be\ndesigning for a requirement neither epic states.\n\n`docs/architecture/plugins-and-extensions.md` does **not** need amending.\nThe line that looked like a blanket ban on hooks (`archie-core-7d5u`'s notes\nrecord the correction) is scoped to Workflow semantics specifically, and\nnothing decided here touches Workflow's ownership of its own contracts.\n\n## 2. Reaction delivery: `internal/eventbus`'s contract, a NEW stream\n\n`internal/eventbus.Publisher`/`Consumer` already has everything a reaction\nneeds: `PublishUnique(subject, idempotencyKey, payload)` for at-least-once\nwithout duplicate enqueue, `Ack`/`Nak` with \"a handler returning an error\ncauses redelivery.\" Do not design new delivery semantics; use this contract.\n\n**Do not reuse the existing `nats.Client` as-is.** `Config.StreamName` and\n`Config.Subjects` are already composition-supplied (\"the bus must not know\nwhich subjects belong to which domain,\" `config.go`), but the\nretention policy is not parameterized: `New` hardcodes\n`Retention: jetstream.WorkQueuePolicy` (`client.go`), the competing-consumer\npolicy correct for `ARCHIE_TASKS` task distribution -- one message, claimed by\nexactly one consumer -- and wrong for reactions. Two independent reactions\nboth wanting to see the same event under `WorkQueuePolicy` hit exactly the\ntrap `CLAUDE.md` already documents: overlapping filter subjects on a\nwork-queue stream mean the second consumer silently gets nothing. Reactions\nneed fan-out, not claim-once, so the client needs retention made a `Config`\nfield (defaulting to `WorkQueuePolicy` so `ARCHIE_TASKS` is\nunaffected), and a second stream (`ARCHIE_REACTIONS` or similar) configured\nunder a fan-out-capable policy, alongside the existing task-distribution\nstream -- not instead of it.\n\nThis is where `7d5u.2`'s embedded-vs-external axis attaches: whichever NATS\ninstance backs `ARCHIE_TASKS` also backs the reaction stream. NATS is not\noptional as a technology -- the choice is only\nwhether it runs as an external server or embedded in-process\n(`github.com/nats-io/nats-server/v2` is already a direct, test-proven\ndependency, just never wired into production startup). `7d5u.2` wires that;\nthis document does not re-decide it.\n\n## 3. Reactions in these two epics are producer-only\n\nNo veto, no mutation of in-flight work. Two existing shapes cover everything\neither epic actually needs, and neither is a hook that can block something\nelse:\n\n- **Cheap wake, no payload.** `internal/domain/curator/wake.go`'s\n `WakeOnPrimaryInput` subscribes to the lossy `events.Bus` and nudges a\n runtime to check sooner; a dropped event only delays a check-in, never\n loses work, because the trigger decision is a deterministic state read at\n pass time, not the event payload. Model any \"check something sooner\"\n reaction on this, unchanged.\n- **Produce new work, real payload.** `daemon.go`'s existing `pollNATS` -\u003e\n `publishTask` path is already exactly a reaction that observes forge state\n and produces a `workintake.TaskEnvelope`. A webhook source is a second\n producer into the same path, not a new capability -- see question 4.\n\nA reaction that can veto or mutate an *in-flight* task (block a PR from\nopening, alter a running stage) is a different, harder problem with its own\nordering and failure semantics, and belongs to `archie-core-h019` (adversarial\nself-review) if and when that epic needs one -- not here. Do not build a\ngeneral block-capable hook mechanism for either epic; nothing in their stated\nscope requires it, and it would duplicate whatever h019 eventually needs.\n\n## 4. No single generic intake interface; each domain keeps its own\n\nA `Source` interface generic enough to cover a forge issue, a chat message,\nand a cron tick would need an `any` payload -- the untyped-hook shape the\nplugin engine rule exists to prevent, just moved into a new package instead\nof the old one.\n\n**What's genuinely shared, and belongs in a small cross-cutting package**\n(imports nothing per the cross-cutting rule): the mechanics from\n`archie-core-t2db.5` that apply to any inbound HTTP source regardless of what\nconsumes it -- HMAC verification (already a standalone function,\n`internal/channels/webhook/webhook.go`'s `validateHMAC`), per-source rate\nlimiting, capture/audit, and the draft/pending_approval/armed gate. This\npackage owns none of forge, chat, or workintake's vocabulary.\n\n**What stays domain-owned, per existing convention:**\n- Forge-issue intake needs no new type. `internal/domain/workintake.TaskEnvelope`\n already is the typed contract -- `Subject()`, `IdempotencyKey()` (keyed on\n `owner/repo/number`, not on delivery source), `Encode()`. A webhook-sourced\n envelope for the same issue produces the *same* idempotency key as a\n poll-discovered one, so `PublishUnique` dedups the two paths for free. This\n is `7d5u.5`'s answer, not a new mechanism: reuse `TaskEnvelope` and\n `publishTask`, don't invent a second envelope shape.\n- Chat-webhook intake stays exactly what it is:\n `internal/channels/webhook`'s `RouteConfig` routes into `gateway.Router`,\n documented as chat-scoped (\"External services push messages into\n archie-core's gateway for LLM processing\"). Do not generalize this package\n into a multi-domain receiver -- it already does one thing per \"a package\n owns its own format end to end.\"\n- A forge webhook receiver (`7d5u.4`) is new: it builds on the shared\n mechanics above, decodes a GitHub payload into the *same*\n `workintake.TaskEnvelope`, and calls the *same* `publishTask` the poller\n calls. It does not get its own dispatch predicate -- `7d5u.4`'s bead\n already requires reusing the poller's label/assignee predicate rather than\n reimplementing it, and this decision extends that to the envelope and\n publish path too.\n\n## Packages this touches\n\n- New: a cross-cutting intake-mechanics package (HMAC, rate limit, capture,\n approval state machine) -- name and exact location decided in `t2db.1`/`t2db.5`.\n- `internal/infrastructure/eventbus/nats`: a second stream/consumer\n configuration for reactions, distinct retention policy from `ARCHIE_TASKS`.\n- `internal/domain/workintake`: unchanged in shape; `TaskEnvelope` is reused,\n not extended.\n- `internal/daemon`: gains a webhook-sourced call path into the existing\n `publishTask`, alongside the existing poll-sourced one.\n- `internal/channels/webhook`: unchanged. Not the forge receiver.\n" + "body": "# Event sources and reactions -- decision\n\n**Status:** Approved\n**Date:** 2026-08-22\n**Beads issue:** `archie-core-7d5u.1`, blocks `7d5u.2/.3/.4/.5`\n\nAnswers the four questions in `7d5u.1`'s description. Investigated against\nthe actual tree, not designed from scratch: three mechanisms already in this\ncodebase answer most of this, and the job here is picking the right one for\neach question and saying which packages change.\n\n## 1. In-process typed families, not out-of-process dispatch\n\nArchie has no out-of-process extension mechanism. The plugin engine\nrule (`docs/architecture/plugins-and-extensions.md#plugin-engine-rule-strict`,\nmechanically checked by\n`internal/plugin/architecture_test.go`) keeps every capability family\nin-process behind a narrow typed registrar; NATS RPC exists for\ndaemon\u003c-\u003eagentexec, a different actor with a different trust boundary (task\nexecution, not third-party extension code), not a precedent for generic\nplugin dispatch. Neither epic's near-term payoff -- react to a labelled\nissue, bind a captured webhook to a workflow -- needs extension code in\nanother language or process. Building out-of-process dispatch now would be\ndesigning for a requirement neither epic states.\n\n`docs/architecture/plugins-and-extensions.md` does **not** need amending.\nThe line that looked like a blanket ban on hooks (`archie-core-7d5u`'s notes\nrecord the correction) is scoped to Workflow semantics specifically, and\nnothing decided here touches Workflow's ownership of its own contracts.\n\n## 2. Reaction delivery: `internal/eventbus`'s contract, a NEW stream\n\n`internal/eventbus.Publisher`/`Consumer` already has everything a reaction\nneeds: `PublishUnique(subject, idempotencyKey, payload)` for at-least-once\nwithout duplicate enqueue, `Ack`/`Nak` with \"a handler returning an error\ncauses redelivery.\" Do not design new delivery semantics; use this contract.\n\n**Do not reuse the existing `nats.Client` as-is.** `Config.StreamName` and\n`Config.Subjects` are already composition-supplied (\"the bus must not know\nwhich subjects belong to which domain,\" `config.go`), but the\nretention policy is not parameterized: `New` hardcodes\n`Retention: jetstream.WorkQueuePolicy` (`client.go`), the competing-consumer\npolicy correct for `ARCHIE_TASKS` task distribution -- one message, claimed by\nexactly one consumer -- and wrong for reactions. Two independent reactions\nboth wanting to see the same event under `WorkQueuePolicy` hit exactly the\ntrap `CLAUDE.md` already documents: overlapping filter subjects on a\nwork-queue stream mean the second consumer silently gets nothing. Reactions\nneed fan-out, not claim-once, so the client needs retention made a `Config`\nfield (defaulting to `WorkQueuePolicy` so `ARCHIE_TASKS` is\nunaffected), and a second stream (`ARCHIE_REACTIONS` or similar) configured\nunder a fan-out-capable policy, alongside the existing task-distribution\nstream -- not instead of it.\n\nThis is where `7d5u.2`'s embedded-vs-external axis attaches: whichever NATS\ninstance backs `ARCHIE_TASKS` also backs the reaction stream. NATS is not\noptional as a technology -- the choice is only\nwhether it runs as an external server or embedded in-process\n(`github.com/nats-io/nats-server/v2` is already a direct, test-proven\ndependency, just never wired into production startup). `7d5u.2` wires that;\nthis document does not re-decide it.\n\n## 3. Reactions in these two epics are producer-only\n\nNo veto, no mutation of in-flight work. Two existing shapes cover everything\neither epic actually needs, and neither is a hook that can block something\nelse:\n\n- **Cheap wake, no payload.** `internal/domain/curator/wake.go`'s\n `WakeOnPrimaryInput` subscribes to the lossy `events.Bus` and nudges a\n runtime to check sooner; a dropped event only delays a check-in, never\n loses work, because the trigger decision is a deterministic state read at\n pass time, not the event payload. Model any \"check something sooner\"\n reaction on this, unchanged.\n- **Produce new work, real payload.** `daemon.go`'s existing `pollNATS` -\u003e\n `publishTask` path is already exactly a reaction that observes forge state\n and produces a `workintake.TaskEnvelope`. A webhook source is a second\n producer into the same path, not a new capability -- see question 4.\n\nA reaction that can veto or mutate an *in-flight* task (block a PR from\nopening, alter a running stage) is a different, harder problem with its own\nordering and failure semantics, and belongs to `archie-core-h019` (adversarial\nself-review) if and when that epic needs one -- not here. Do not build a\ngeneral block-capable hook mechanism for either epic; nothing in their stated\nscope requires it, and it would duplicate whatever h019 eventually needs.\n\n## 4. No single generic intake interface; each domain keeps its own\n\nA `Source` interface generic enough to cover a forge issue, a chat message,\nand a cron tick would need an `any` payload -- the untyped-hook shape the\nplugin engine rule exists to prevent, just moved into a new package instead\nof the old one.\n\n**What's genuinely shared, and belongs in a small cross-cutting package**\n(imports nothing per the cross-cutting rule): the mechanics from\n`archie-core-t2db.5` that apply to any inbound HTTP source regardless of what\nconsumes it -- HMAC verification (already a standalone function,\n`internal/channels/webhook/webhook.go`'s `validateHMAC`), per-source rate\nlimiting, capture/audit, and the draft/pending_approval/armed gate. This\npackage owns none of forge, chat, or workintake's vocabulary.\n\n**What stays domain-owned, per existing convention:**\n- Forge-issue intake needs no new type. `internal/domain/workintake.TaskEnvelope`\n already is the typed contract -- `Subject()`, `IdempotencyKey()` (keyed on\n `owner/repo/number`, not on delivery source), `Encode()`. A webhook-sourced\n envelope for the same issue produces the *same* idempotency key as a\n poll-discovered one, so `PublishUnique` dedups the two paths for free. This\n is `7d5u.5`'s answer, not a new mechanism: reuse `TaskEnvelope` and\n `publishTask`, don't invent a second envelope shape.\n- Chat-webhook intake stays exactly what it is:\n `internal/channels/webhook`'s `RouteConfig` routes into `gateway.Router`,\n documented as chat-scoped (\"External services push messages into\n archie-core's gateway for LLM processing\"). Do not generalize this package\n into a multi-domain receiver -- it already does one thing per \"a package\n owns its own format end to end.\"\n- A forge webhook receiver (`7d5u.4`) is new: it builds on the shared\n mechanics above, decodes a GitHub payload into the *same*\n `workintake.TaskEnvelope`, and calls the *same* `publishTask` the poller\n calls. It does not get its own dispatch predicate -- `7d5u.4`'s bead\n already requires reusing the poller's label/assignee predicate rather than\n reimplementing it, and this decision extends that to the envelope and\n publish path too.\n\n## Packages this touches\n\n- New: a cross-cutting intake-mechanics package (HMAC, rate limit, capture,\n approval state machine) -- name and exact location decided in `t2db.1`/`t2db.5`.\n- `internal/infrastructure/eventbus/nats`: a second stream/consumer\n configuration for reactions, distinct retention policy from `ARCHIE_TASKS`.\n- `internal/domain/workintake`: unchanged in shape; `TaskEnvelope` is reused,\n not extended.\n- `internal/daemon`: gains a webhook-sourced call path into the existing\n `publishTask`, alongside the existing poll-sourced one.\n- `internal/channels/webhook`: unchanged. Not the forge receiver.\n" }, { "url": "/dev/prds/extensions-center/", @@ -256,7 +256,7 @@ "url": "/dev/prds/image-capability-contract/", "title": "Image capability contract and configuration", "section": "PRDs", - "body": "# Image capability contract and configuration\n\n**Status:** Approved\n\nEpic: archie-core-1786748942120-1-6636e629 / GitHub #496 (\"Image generation\ncapability: hosted API baseline plus local GPU workflows\").\n\nThe contract and configuration (#497) shipped first and is settled — see\n\"Package: `internal/domain/image`\" below. This document now covers the full\nfeature set: hosted provider (#498), local GPU provider (#499), `/image`\nrouting and clarification (#500), delivery through channels and web (#501).\nDocumentation (#502) and the cross-cutting test matrix (#503) are execution\nwork against the design below, not open design questions, so they are not\ngiven their own sections — each preceding section states its own test\nobligations inline.\n\n## Problem\n\nArchie needs a provider-neutral boundary for image generation/editing so a\nhosted API (gpt-image-2-class) and a future local GPU backend can both\nimplement it without either one leaking into the domain contract. Nothing\nin the repo defines this (`grep -ri image` over `internal/domain` and\n`internal/config` returns nothing image-specific).\n\n## Design\n\nFollows the plugin engine family shape already established by\n`internal/domain/curator` and `internal/domain/memory` (ARCHITECTURE.md\n\"plugin engine rule strict\"): a typed contract with real domain operations,\nan owning `Registry` (register/get/names, no start/stop lifecycle — see\nbelow), and narrow host access via a `Registrar`.\n\n### Package: `internal/domain/image`\n\n**Why no `Lifecycle` (`Start/Health/Stop`)**: curator and memory engines are\nlong-running — they hold a background cadence or a persistent connection\nworth health-checking. An image provider is call-scoped: it either serves a\n`Generate`/`Edit` request or it does not. Health here means \"is this\nprovider usable at call time\", which is exactly what `Generate`/`Edit`\nreturning a typed `ErrUnavailable` already communicates — a separate\n`Health()` poll would just duplicate that. `Registry` still owns\nregistration and lookup, so provider selection is testable with fakes per\nthe acceptance criteria, but there is no `Start`/`Stop` to orchestrate.\n\n```go\n// Capability declares what one provider supports, discoverable before a\n// request is built so a caller can reject an unsupported combination (e.g.\n// edit-with-mask on a generate-only provider) instead of sending it and\n// parsing a provider-specific error back out.\ntype Capability struct {\n Generate bool\n Edit bool // edit requires at least one input image\n Mask bool // edit supports a mask restricting the edited region\n MaxInputs int // 0 = Edit unsupported\n Sizes []string // provider-declared output sizes, e.g. \"1024x1024\"\n}\n\n// Provider is the typed contract every image backend implements — hosted\n// (gpt-image-2-class) or local (GPU recipe). No Lifecycle: a provider is\n// call-scoped, not long-running (see design doc rationale).\ntype Provider interface {\n Name() string\n // Class distinguishes hosted (spends money/quota per call) from local\n // (spends host GPU/time). Selection logic and config validation key off\n // this, never off Name, so a new provider cannot silently slip through\n // the hosted-cost gate under an unrecognized name.\n Class() ProviderClass\n Capability() Capability\n Generate(ctx context.Context, req GenerateRequest) (Result, error)\n Edit(ctx context.Context, req EditRequest) (Result, error)\n}\n\ntype ProviderClass string\n\nconst (\n ClassHosted ProviderClass = \"hosted\"\n ClassLocal ProviderClass = \"local\"\n)\n```\n\nRequest/result types (provider-neutral, no OpenAI types):\n\n```go\ntype GenerateRequest struct {\n Prompt string\n Size string // \"\" = provider default\n N int // number of images; 0 = 1\n Options map[string]string // provider-specific overflow (e.g. \"quality\"),\n // deliberately untyped so the contract does not\n // grow a field per provider\n}\n\ntype EditRequest struct {\n Prompt string\n Inputs []ImageData // 1..Capability.MaxInputs\n Mask *ImageData // nil unless Capability.Mask\n Size string\n N int\n Options map[string]string\n}\n\n// ImageData carries image bytes in-process, never a path or URL: the\n// domain layer must not assume a filesystem or a fetchable location (mirrors\n// why curator's Registrar never receives *daemon.Daemon — no ambient host\n// access smuggled in through a \"just a string\" field).\ntype ImageData struct {\n Bytes []byte\n MIMEType string\n}\n\ntype Result struct {\n Images []ImageData\n Provider string\n Model string\n CreatedAt time.Time\n}\n```\n\n### Error taxonomy\n\n```go\nvar (\n ErrUnavailable = errors.New(\"image provider unavailable\") // missing credential, backend down\n ErrUnsupported = errors.New(\"image capability unsupported by provider\") // e.g. Edit on generate-only\n ErrInputTooLarge = errors.New(\"image input exceeds configured limit\")\n ErrTimeout = errors.New(\"image request timed out\")\n ErrCanceled = errors.New(\"image request canceled\")\n)\n```\n\nEvery provider wraps these with `fmt.Errorf(\"%w: %s\", ErrX, detail)` so\ncallers can `errors.Is` regardless of backend. `ErrUnavailable` is the\nexplicit instruction from the epic: \"a missing credential or unavailable\nbackend must make the capability unavailable with a useful user-facing\nerror, not prevent Archie from starting or break text chat\" — Register\nnever rejects the daemon's boot, only marks that one provider absent from\n`Registry.Names()`.\n\n### Registry\n\n```go\ntype Registry struct { /* map[string]Provider, mutex, sorted Names() */ }\n\nfunc NewRegistry() *Registry\nfunc (r *Registry) Register(p Provider) error // ErrDuplicate on name clash\nfunc (r *Registry) Get(name string) (Provider, bool)\nfunc (r *Registry) Names() []string\nfunc (r *Registry) ByClass(class ProviderClass) []string\n```\n\nNo `Registrar`/host-injection type: unlike curator (needs `LLM`, `Skills`,\n`Conversations`) and memory (needs `Clock`, `Events`), an image provider's\ndependencies (HTTP client, API key, local backend handle) are entirely its\nown construction-time concern in `internal/infrastructure/image/*`, not\ndaemon services it needs injected. Introducing an empty `Registrar` now\nwould be exactly the \"field nobody sets\" pattern to avoid.\n\n### Configuration\n\nFollows `MinimaxConfig`'s hosted-cost-guard convention exactly: absent\nconfig or absent explicit `enabled = true` per provider means that\nprovider does not register, not \"registers but errors on first call\".\n\n```go\n// ImageConfig configures the image generation capability.\ntype ImageConfig struct {\n // Default selects which registered provider name /image uses when the\n // request/session has not already picked one. Empty means /image must\n // always ask (the epic's stated default UX) — an empty Default is valid\n // and is not a misconfiguration.\n Default string `toml:\"default\"`\n Hosted map[string]ImageHostedProvider `toml:\"hosted\"` // keyed by provider name\n Local map[string]ImageLocalProvider `toml:\"local\"` // keyed by recipe name\n}\n\n// ImageHostedProvider configures one hosted (paid) provider. Mirrors\n// MinimaxConfig: Enabled defaults false, spending real credits requires an\n// explicit opt-in, never an inferred one from the presence of a key alone.\ntype ImageHostedProvider struct {\n Enabled bool `toml:\"enabled\"`\n Class string `toml:\"class\"` // \"openai\" today; provider-neutral field name\n APIKeyEnv string `toml:\"api_key_env\"`\n APIKey secret.SecretRef `toml:\"api_key\" json:\"-\"`\n BaseURL string `toml:\"base_url\"`\n}\n\n// ImageLocalProvider configures one local GPU recipe. Enabled defaults\n// false too: a local backend can still saturate the host GPU/CPU, which is\n// its own kind of unwanted cost, so it gets the same explicit gate as a\n// hosted provider rather than an on-by-default WebFetch-style treatment.\ntype ImageLocalProvider struct {\n Enabled bool `toml:\"enabled\"`\n Backend string `toml:\"backend\"` // recipe identifier, defined by the local-backend bead\n}\n```\n\nValidation (`internal/infrastructure/configuration/validate.go`, alongside\n`validateMemory`):\n\n- `Default`, if set, must name a provider present (enabled) in `Hosted` or\n `Local` — rejects a config that silently falls back to \"always ask\" when\n the operator thought they'd pinned one.\n- A `Hosted` entry with `Enabled = true` must have `Class` non-empty and\n either `APIKeyEnv` or a resolvable `APIKey` — the same shape check\n `Provider` (chat) already gets, not a new invention.\n- No cross-field surprise: an *absent* `[image]` section is valid and\n produces zero registered providers, matching the epic's non-goal\n (\"silently selecting a paid hosted provider\" must never happen by\n omission either).\n\n### Capability discovery\n\n`Registry.Names()` / `ByClass()` plus `Provider.Capability()` give a caller\n(the future `/image` routing bead) everything needed to ask \"what can I\noffer this user\" without a type switch on provider identity — this is the\nwhole point of the contract being provider-neutral.\n\n### Timeouts, size limits, cancellation\n\n- Every `Provider.Generate`/`Edit` call takes the caller's `ctx`; a\n provider that ignores cancellation is a provider bug, not a contract gap\n — mirrors every other Lifecycle-less call in the codebase (chat's\n `runtime.Chat`).\n- `ImageData` size limits are enforced by the *caller* (the future routing\n bead) before constructing a request, using a configured max — not by this\n contract, which has no config surface for \"current caller's limit\". This\n package only defines `ErrInputTooLarge` so a provider that discovers an\n oversized input itself (rather than trusting the caller) has a typed way\n to say so.\n\n## Testing\n\n- `internal/domain/image/{contract,registry}_test.go` mirror memory's test\n suite shape: registration (duplicate, nil provider), `Names`/`ByClass`\n filtering, `Get` miss, table-driven `Capability` checks via a fake\n provider.\n- `internal/infrastructure/configuration/validate_test.go` gains cases for\n `ImageConfig`: absent section, valid hosted+local, missing `Class`,\n missing credential, `Default` naming an unregistered/disabled provider.\n- No provider implementation exists yet, so no live-call tests — those\n belong to the hosted/local provider beads.\n\n## Hosted provider (#498)\n\n`github.com/samcharles93/ai-sdk` (already a repo dependency) has its own\nprovider-agnostic image facility: `image.Provider` (`Name`,\n`GenerateImage(ctx, GenerateImageRequest) (GenerateImageResponse, error)`),\n`image.Client`, and `registry.Registry.RegisterImage`/`.Image(name)` — the\nsame shape chat model registration already uses. Three backends implement it\n(azure, togetherai, xai); OpenAI does not.\n\n`internal/infrastructure/image.\u003cname\u003eProvider` wraps an `ai-sdk/image.Provider`\nbehind this package's `Provider` interface, the same way `curatorLLMRunner`\n(`internal/app/archied/main.go`) adapts `runtime.Runtime` for chat rather\nthan the domain layer talking to ai-sdk directly.\n\n**Decision: generation goes through ai-sdk, edit does not.** ai-sdk's\n`image.GenerateImageRequest` is text-to-image only — no input-image or mask\nfields — so there is nothing to adapt `EditRequest`/`Capability.Edit` onto.\nBlocking the entire hosted provider on an upstream ai-sdk change is not\nthis epic's call to make on someone else's project's timeline. Instead:\n\n- `Generate` wraps `ai-sdk/image.Client.GenerateImage` against whichever\n backend `ImageHostedProvider.Class` names (azure/togetherai/xai).\n- `Edit` is a direct HTTP client hitting the OpenAI-compatible\n `/images/edits` multipart endpoint, built the same way `minimax.Client`\n (`internal/tools/minimax/minimax.go`) is a direct client for an API\n ai-sdk doesn't cover — not a second abstraction layer, one `http.Client`\n call with request/response mapping into this contract's `EditRequest`/\n `Result`.\n- A provider that only wraps ai-sdk (no edit client configured) declares\n `Capability{Generate: true, Edit: false}` truthfully rather than\n returning `ErrUnsupported` at call time for something `Capability`\n already said it couldn't do.\n- If ai-sdk gains edit support upstream later, the direct client is deleted\n and `Edit` moves onto it — recorded here so that's a known, not a\n surprise, follow-up.\n\n**Provider error mapping** (every ai-sdk/HTTP error translated at the\nadapter boundary, never leaked as-is):\n\n| ai-sdk / HTTP condition | contract error |\n|---|---|\n| `image.ErrNoProvider`, `image.ErrAuthFailed`, missing/invalid credential, backend unreachable | `ErrUnavailable` |\n| `image.ErrEditNotSupported`, request needs a `Capability` the provider didn't declare | `ErrUnsupported` |\n| input image over configured byte limit (checked before the call, per \"Timeouts, size limits, cancellation\" above) | `ErrInputTooLarge` |\n| `ctx.Err() == context.DeadlineExceeded` | `ErrTimeout` |\n| `ctx.Err() == context.Canceled` | `ErrCanceled` |\n| `image.ErrRateLimited`, `image.ErrContentFiltered`, any other 4xx/5xx | wrapped verbatim under `ErrUnavailable` with the provider's own message as detail — a rate limit or content filter is \"unavailable for this request\", not a new sentinel the caller needs to branch on differently |\n\n**Testing**: `httptest.Server` fixtures for the edit client (request\nmultipart shape, response decoding, 4xx/5xx mapping); a fake `ai-sdk/image.Provider`\nfor the generate path so the adapter's translation is tested without a live\nbackend, matching #498's \"no live API key needed for tests\" criterion.\n\n## Local GPU provider (#499)\n\n**Decision: ComfyUI is the baseline recipe**, not a name this doc picked\narbitrarily — it is the de facto scriptable HTTP API for Stable-Diffusion-class\nlocal generation, already shaped like the async submit/poll pattern\n`minimax.Client.GenerateAndWait` established in this codebase (submit a\nworkflow, poll `/history/{id}` until the output node has files, download).\nNo other local backend is referenced anywhere in this repo or its docs, so\nthis decision has no competing prior art to reconcile — it is a fresh\nchoice, recorded here so the implementing bead does not re-litigate it.\n\n```go\n// internal/infrastructure/image/comfyui\ntype Config struct {\n Enabled bool\n BaseURL string // e.g. http://127.0.0.1:8188\n Workflow string // path to a saved ComfyUI workflow JSON template\n // (prompt/size/seed substituted at call time)\n Timeout time.Duration\n PollInterval time.Duration\n MaxWait time.Duration\n}\n```\n\n- `Generate`/`Edit` load `Workflow`, substitute the templated fields (prompt,\n size, seed, and for `Edit` the input image as a base64 node input — ComfyUI\n supports img2img/inpaint workflows natively, so `Capability.Edit`/`Mask`\n can both be true for a correctly authored workflow, unlike the hosted\n provider), POST to `/prompt`, then poll `/history/{prompt_id}` the same\n shape MiniMax's poll loop already proves out in this codebase.\n- `Capability.Sizes` is empty (ComfyUI workflows declare their own\n resolution in the template, not a fixed enum) — the contract already\n allows an empty `Sizes` to mean \"provider does not constrain size\".\n- Backend-down (`BaseURL` unreachable) and workflow-missing are both\n `ErrUnavailable`, not a startup failure — same \"degrade the capability,\n never break text chat\" rule the contract already states.\n- Per #499's own acceptance criteria, at least one recipe must be exercised\n end-to-end on real hardware or a reproducible harness, and no model\n weights are ever committed or fetched by Archie itself — the operator\n points `Workflow`/`BaseURL` at their own running ComfyUI instance and\n already-downloaded checkpoint, exactly like `[providers.ollama]` points at\n an already-running Ollama rather than Archie managing the model.\n\n**Testing**: a fake HTTP server standing in for ComfyUI's `/prompt` and\n`/history` endpoints (submit, poll, malformed history, timeout, job\nnot-found); a real-hardware smoke test documented as a manual `task`\ntarget per #503, not part of `task check` — the same reasoning `task vuln`\nand `task deadcode` are excluded from the gate.\n\n## `/image` routing and clarification flow (#500)\n\n**Call site: `internal/gateway/gateway.go`'s `Router`**, following the\n`/model` shape exactly — `/model`, `/spawn`, `/approve` etc. are already\n\"commands handled without the LLM\" (`Router.handleModel`, dispatched via\n`isLocalCommand`/`parseCmd` against `messaging.LocalCommandSpecs()`).\n`/image` is a new entry in `internal/domain/messaging/commands.go`'s\n`localCommandSpecs`, and a new `Router.handleImage(ctx, arg)` beside\n`handleModel`, gated on a new `Router.Images *image.Registry` field (nil =\n\"/image not configured\", the same pattern `Models`/`Tasks`/`Controller`\nalready use — not a new gating convention).\n\n**Decision: mechanism selection is per-request, not session-persisted.**\nThe epic allows \"a remembered preference only where existing session/config\nconventions support it\" — and checking `Router`'s actual fields\n(`ModelPersist`, `Identity`, the rest) shows there is no generic per-session\nkey/value store the neutral command path can lean on; `/model`'s own\n`--session`/`--global` scoping lives entirely in\n`internal/channels/telegram/model.go`, a Telegram-specific enrichment, not\nsomething `Router` provides generically. Inventing a new generic\nsession-preference store just for `/image` would be exactly the kind of\nscaffolding CLAUDE.md's Architectural Simplicity rule warns against for one\ncaller. So: every `/image` invocation with no explicit mechanism produces\nthe clarification prompt; an explicit `--hosted`/`--local` flag (mirroring\n`parseModelRest`'s `--provider` flag shape) skips it for that one call.\nRevisit if a second command wants session-scoped preference too — that's\nthe point at which shared infra earns its cost.\n\n**Flow**, matching `handleModel`'s reply-string shape (no separate state\nmachine object, since the whole exchange is \"ask once, use the answer\ninline in the next message\" — the model-selector's own worked example):\n\n1. `/image \u003cprompt\u003e` with no `--hosted`/`--local` → reply lists the\n registered classes (from `Registry.ByClass`) and asks the user to repeat\n the command with `--hosted` or `--local`. No provider is called.\n2. `/image --hosted \u003cprompt\u003e` or `--local` → resolves to\n `Registry.ByClass(class)`, picks `ImageConfig.Default` if it names a\n provider of that class, else the sole registered provider of that class,\n else (more than one, no default) asks which named provider.\n3. Edit requests (`/image --edit \u003cprompt\u003e` with an attached image on the\n inbound message) validate against `Capability.Edit`/`MaxInputs`/`Mask`\n before calling — an edit request to a generate-only provider is a\n clarification reply (\"this provider doesn't support editing\"), not a\n provider call that returns `ErrUnsupported`.\n4. A successful call returns a `Result`; delivery is the next section's\n concern, not this one's — `handleImage` returns the same\n `(string, error)` shape `handleModel` does for the text portion, and a\n separate mechanism (below) carries the actual image bytes.\n\n**Testing**: channel-neutral state-machine tests for `handleImage` against\na fake `*image.Registry` (mirrors `#500`'s own acceptance criterion); Telegram\nadapter tests only for wiring, not for re-testing the state machine.\n\n## Delivery through channels and web (#501)\n\nThis is the smallest section because most of the delivery path **already\nexists and is already proven** — by `generate_video`\n(`internal/tools/minimax`), which ships. Do not rebuild it.\n\nThe existing pipeline: a result carrying media returns\n`tools.MultimodalResult{IsMultimodal: true, URLs: []tools.MediaRef{{Type: \"image\", ...}}}`\n→ `multimodalMediaRefs` (`internal/app/archied/telegram_setup.go`) decodes\nit → `TurnStream.Media(gateway.MediaEvent)` → each channel's own\n`Media`/`MediaAttachment` handling. `MediaRef.Type` already documents\n`\"image\"` as a valid value alongside `\"video\"`; nothing in the pipeline is\nvideo-specific.\n\n**Per-channel state, checked directly rather than assumed:**\n\n| channel | image delivery | evidence |\n|---|---|---|\n| Telegram | **already works** | `internal/channels/telegram/media.go`'s `telegramMediaSender.dispatch` already branches on `att.Type` and calls `bot.SendPhoto` for `\"image\"` (`SendVideo` for `\"video\"` is the same function) |\n| Dashboard (webui) | **tracked gap, not new** | `internal/webui/api_chat.go`'s `chatStreamSink.Media` has no inline rendering path yet and degrades to a link/error string — already tracked as archie-core-1786748942243-6-f109697e. This epic does not need to fix it, but #501's acceptance criterion (\"unsupported paths report a clear limitation\") is already met by the existing degrade-to-text behavior; closing the dashboard gap is that bead's job, not this one's |\n| Email | **no attachment sender exists** | `grep MediaAttachment internal/channels/email` returns nothing. Out of scope for this epic per #496's non-goals (no channel-abstraction rework) — email delivery of a generated image degrades the same way the dashboard does which is acceptable under #501's \"or unsupported paths report a clear limitation\" clause |\n| Webhook | not applicable | webhook is an inbound intake channel (`internal/channels/webhook`), not an outbound chat reply target — no `/image` command can originate there in the first place |\n\n**The only new work `internal/tools/minimax` didn't already need**: the\nautonomous-tool call site. `/image` (previous section) calls a provider\ndirectly and does not go through the model's tool-calling loop at all.\nSeparately, exposing `generate_image`/`edit_image` as an ordinary tool\n(so an agent task can generate a diagram or illustration mid-task, not only\na human typing `/image`) is a thin wrapper — `internal/tools/image/tool.go`,\nsame shape as `internal/tools/minimax/tool.go`, calling\n`image.Registry.Get(name).Generate/Edit` and returning the identical\n`MultimodalResult` shape. Wired in `bootstrap.go` next to\n`registerMinimaxTool`, gated the same way (registered only when a provider\nresolved a working credential/backend, never advertised broken). This\nreuses the entire delivery pipeline above with zero new plumbing — the tool\ncall site is what's new, not the delivery.\n\n**Testing**: `multimodalMediaRefs` already has coverage for the shape;\nadd a table case for `Type: \"image\"` if one doesn't already exist. The tool\nwrapper gets the same fake-provider test shape as the routing flow above.\nNo new delivery-path tests are needed — that pipeline's tests belong to\nwhichever change originally proved it out.\n\n## Call site inventory\n\nEvery file a full implementation touches, so no bead re-derives this by\ngrep:\n\n| concern | file | change |\n|---|---|---|\n| domain contract | `internal/domain/image/{contract,registry}.go` | done (#497) |\n| config schema + validation | `internal/infrastructure/configuration/{types,validate}.go` | done (#497); extended here only if ComfyUI's `Workflow` field needs a new validation rule (path exists, non-empty when `Local.Enabled`) |\n| hosted provider | `internal/infrastructure/image/openai/*.go` (or per `Class`) | new (#498) |\n| local provider | `internal/infrastructure/image/comfyui/*.go` | new (#499) |\n| `/image` command spec | `internal/domain/messaging/commands.go` | new entry in `localCommandSpecs` |\n| `/image` routing | `internal/gateway/gateway.go` (`Router.Images`, `handleImage`) | new (#500) |\n| autonomous tool | `internal/tools/image/tool.go` | new (#501) |\n| tool registration | `internal/app/archied/bootstrap.go` (`registerImageTool`, beside `registerMinimaxTool`) | new (#501) |\n| `Registry` wiring | `internal/app/archied/bootstrap.go` (construct `image.Registry`, register configured providers, hand to `Router.Images`) | new — this is the \"nothing to wire until a real provider exists\" step the original contract doc deferred |\n| Telegram delivery | `internal/channels/telegram/media.go` | none — already handles `\"image\"` |\n| Dashboard delivery | `internal/webui/api_chat.go` | none in this epic — tracked separately (archie-core-1786748942243-6-f109697e) |\n| docs | `docs/` user-facing guide (#502) | new, against this design |\n\n## Execution: multi-agent team breakdown\n\nEach row is independently implementable from its own section above plus the\ncall-site inventory — an implementer does not need this whole document\nre-explained, only its row. Lenses are picked from\n`.claude/workflows/council.js` (`boundary`, `contract`, `deletionist`,\n`operator`, `maintainer`) by what each slice actually risks getting wrong,\nnot all five by default.\n\n| sub-feature | issue | implementer scope (call-site rows) | suggested lenses | why |\n|---|---|---|---|---|\n| Hosted provider | #498 | `internal/infrastructure/image/openai/*.go` (generate via ai-sdk, edit via direct HTTP client) | `lens-contract`, `lens-operator` | contract: the error-mapping table must actually produce the sentinels it claims, not leak ai-sdk/HTTP errors raw; operator: a live paid API's rate-limit/timeout/auth-failure paths are exactly the 3am-unattended-host case this lens exists for |\n| Local GPU provider | #499 | `internal/infrastructure/image/comfyui/*.go` (submit/poll against ComfyUI) | `lens-operator`, `lens-deletionist` | operator: backend-down, workflow-missing, and stuck-job failure modes on a host the operator's GPU actually runs; deletionist: a second, unevidenced \"recipe\" added later without real-hardware proof is exactly the speculative scaffolding this lens rejects |\n| `/image` routing | #500 | `internal/domain/messaging/commands.go`, `internal/gateway/gateway.go` (`Router.Images`, `handleImage`) | `lens-boundary`, `lens-maintainer` | boundary: a new `Router` field and its dependency on `image.Registry` must respect the same layering `Models`/`Tasks`/`Controller` already do; maintainer: this is the first local command with an explicit ask-once clarification exchange, not a single-shot reply — legibility for the next person who adds a command matters here |\n| Delivery + autonomous tool | #501 | `internal/tools/image/tool.go`, `internal/app/archied/bootstrap.go` (`registerImageTool`) | `lens-deletionist`, `lens-contract` | deletionist: the whole point of this section is \"reuse `generate_video`'s pipeline, do not rebuild it\" — this lens is the check that the implementation actually did that; contract: the new tool's `MultimodalResult`/`MediaRef{Type: \"image\"}` output must match the shape `multimodalMediaRefs` already decodes, not a near-miss |\n\nRun `/council --lenses \u003cpicked\u003e` once a row's implementation is gate-clean\n(`task check` passing), before opening its PR — same timing as any other\nopen-decision residue at the end of a session, not a standing per-commit\ngate.\n\n## File and link the beads\n\n`#498`–`#501` already exist (bd `archie-core-1786748942169-3-ae7cfaea`,\n`…-219-5-9844a72e`, `…-193-4-5072b564`, `…-243-6-f109697e`), filed under\nepic #496 before this document's decisions were made. Each one's\ndescription now links back to its matching section above instead of\nrestating it, and carries its row from the Execution table above so picking\nit up needs no round-trip back to this document.\n" + "body": "# Image capability contract and configuration\n\n**Status:** Approved\n\nEpic: archie-core-1786748942120-1-6636e629 / GitHub #496 (\"Image generation\ncapability: hosted API baseline plus local GPU workflows\").\n\nThe contract and configuration (#497) shipped first and is settled — see\n\"Package: `internal/domain/image`\" below. This document now covers the full\nfeature set: hosted provider (#498), local GPU provider (#499), `/image`\nrouting and clarification (#500), delivery through channels and web (#501).\nDocumentation (#502) and the cross-cutting test matrix (#503) are execution\nwork against the design below, not open design questions, so they are not\ngiven their own sections — each preceding section states its own test\nobligations inline.\n\n## Problem\n\nArchie needs a provider-neutral boundary for image generation/editing so a\nhosted API (gpt-image-2-class) and a future local GPU backend can both\nimplement it without either one leaking into the domain contract. Nothing\nin the repo defines this (`grep -ri image` over `internal/domain` and\n`internal/config` returns nothing image-specific).\n\n## Design\n\nFollows the plugin engine family shape already established by\n`internal/domain/curator` and `internal/domain/memory`\n(`docs/architecture/plugins-and-extensions.md#plugin-engine-rule-strict`): a\ntyped contract with real domain operations,\nan owning `Registry` (register/get/names, no start/stop lifecycle — see\nbelow), and narrow host access via a `Registrar`.\n\n### Package: `internal/domain/image`\n\n**Why no `Lifecycle` (`Start/Health/Stop`)**: curator and memory engines are\nlong-running — they hold a background cadence or a persistent connection\nworth health-checking. An image provider is call-scoped: it either serves a\n`Generate`/`Edit` request or it does not. Health here means \"is this\nprovider usable at call time\", which is exactly what `Generate`/`Edit`\nreturning a typed `ErrUnavailable` already communicates — a separate\n`Health()` poll would just duplicate that. `Registry` still owns\nregistration and lookup, so provider selection is testable with fakes per\nthe acceptance criteria, but there is no `Start`/`Stop` to orchestrate.\n\n```go\n// Capability declares what one provider supports, discoverable before a\n// request is built so a caller can reject an unsupported combination (e.g.\n// edit-with-mask on a generate-only provider) instead of sending it and\n// parsing a provider-specific error back out.\ntype Capability struct {\n Generate bool\n Edit bool // edit requires at least one input image\n Mask bool // edit supports a mask restricting the edited region\n MaxInputs int // 0 = Edit unsupported\n Sizes []string // provider-declared output sizes, e.g. \"1024x1024\"\n}\n\n// Provider is the typed contract every image backend implements — hosted\n// (gpt-image-2-class) or local (GPU recipe). No Lifecycle: a provider is\n// call-scoped, not long-running (see design doc rationale).\ntype Provider interface {\n Name() string\n // Class distinguishes hosted (spends money/quota per call) from local\n // (spends host GPU/time). Selection logic and config validation key off\n // this, never off Name, so a new provider cannot silently slip through\n // the hosted-cost gate under an unrecognized name.\n Class() ProviderClass\n Capability() Capability\n Generate(ctx context.Context, req GenerateRequest) (Result, error)\n Edit(ctx context.Context, req EditRequest) (Result, error)\n}\n\ntype ProviderClass string\n\nconst (\n ClassHosted ProviderClass = \"hosted\"\n ClassLocal ProviderClass = \"local\"\n)\n```\n\nRequest/result types (provider-neutral, no OpenAI types):\n\n```go\ntype GenerateRequest struct {\n Prompt string\n Size string // \"\" = provider default\n N int // number of images; 0 = 1\n Options map[string]string // provider-specific overflow (e.g. \"quality\"),\n // deliberately untyped so the contract does not\n // grow a field per provider\n}\n\ntype EditRequest struct {\n Prompt string\n Inputs []ImageData // 1..Capability.MaxInputs\n Mask *ImageData // nil unless Capability.Mask\n Size string\n N int\n Options map[string]string\n}\n\n// ImageData carries image bytes in-process, never a path or URL: the\n// domain layer must not assume a filesystem or a fetchable location (mirrors\n// why curator's Registrar never receives *daemon.Daemon — no ambient host\n// access smuggled in through a \"just a string\" field).\ntype ImageData struct {\n Bytes []byte\n MIMEType string\n}\n\ntype Result struct {\n Images []ImageData\n Provider string\n Model string\n CreatedAt time.Time\n}\n```\n\n### Error taxonomy\n\n```go\nvar (\n ErrUnavailable = errors.New(\"image provider unavailable\") // missing credential, backend down\n ErrUnsupported = errors.New(\"image capability unsupported by provider\") // e.g. Edit on generate-only\n ErrInputTooLarge = errors.New(\"image input exceeds configured limit\")\n ErrTimeout = errors.New(\"image request timed out\")\n ErrCanceled = errors.New(\"image request canceled\")\n)\n```\n\nEvery provider wraps these with `fmt.Errorf(\"%w: %s\", ErrX, detail)` so\ncallers can `errors.Is` regardless of backend. `ErrUnavailable` is the\nexplicit instruction from the epic: \"a missing credential or unavailable\nbackend must make the capability unavailable with a useful user-facing\nerror, not prevent Archie from starting or break text chat\" — Register\nnever rejects the daemon's boot, only marks that one provider absent from\n`Registry.Names()`.\n\n### Registry\n\n```go\ntype Registry struct { /* map[string]Provider, mutex, sorted Names() */ }\n\nfunc NewRegistry() *Registry\nfunc (r *Registry) Register(p Provider) error // ErrDuplicate on name clash\nfunc (r *Registry) Get(name string) (Provider, bool)\nfunc (r *Registry) Names() []string\nfunc (r *Registry) ByClass(class ProviderClass) []string\n```\n\nNo `Registrar`/host-injection type: unlike curator (needs `LLM`, `Skills`,\n`Conversations`) and memory (needs `Clock`, `Events`), an image provider's\ndependencies (HTTP client, API key, local backend handle) are entirely its\nown construction-time concern in `internal/infrastructure/image/*`, not\ndaemon services it needs injected. Introducing an empty `Registrar` now\nwould be exactly the \"field nobody sets\" pattern to avoid.\n\n### Configuration\n\nFollows `MinimaxConfig`'s hosted-cost-guard convention exactly: absent\nconfig or absent explicit `enabled = true` per provider means that\nprovider does not register, not \"registers but errors on first call\".\n\n```go\n// ImageConfig configures the image generation capability.\ntype ImageConfig struct {\n // Default selects which registered provider name /image uses when the\n // request/session has not already picked one. Empty means /image must\n // always ask (the epic's stated default UX) — an empty Default is valid\n // and is not a misconfiguration.\n Default string `toml:\"default\"`\n Hosted map[string]ImageHostedProvider `toml:\"hosted\"` // keyed by provider name\n Local map[string]ImageLocalProvider `toml:\"local\"` // keyed by recipe name\n}\n\n// ImageHostedProvider configures one hosted (paid) provider. Mirrors\n// MinimaxConfig: Enabled defaults false, spending real credits requires an\n// explicit opt-in, never an inferred one from the presence of a key alone.\ntype ImageHostedProvider struct {\n Enabled bool `toml:\"enabled\"`\n Class string `toml:\"class\"` // \"openai\" today; provider-neutral field name\n APIKeyEnv string `toml:\"api_key_env\"`\n APIKey secret.SecretRef `toml:\"api_key\" json:\"-\"`\n BaseURL string `toml:\"base_url\"`\n}\n\n// ImageLocalProvider configures one local GPU recipe. Enabled defaults\n// false too: a local backend can still saturate the host GPU/CPU, which is\n// its own kind of unwanted cost, so it gets the same explicit gate as a\n// hosted provider rather than an on-by-default WebFetch-style treatment.\ntype ImageLocalProvider struct {\n Enabled bool `toml:\"enabled\"`\n Backend string `toml:\"backend\"` // recipe identifier, defined by the local-backend bead\n}\n```\n\nValidation (`internal/infrastructure/configuration/validate.go`, alongside\n`validateMemory`):\n\n- `Default`, if set, must name a provider present (enabled) in `Hosted` or\n `Local` — rejects a config that silently falls back to \"always ask\" when\n the operator thought they'd pinned one.\n- A `Hosted` entry with `Enabled = true` must have `Class` non-empty and\n either `APIKeyEnv` or a resolvable `APIKey` — the same shape check\n `Provider` (chat) already gets, not a new invention.\n- No cross-field surprise: an *absent* `[image]` section is valid and\n produces zero registered providers, matching the epic's non-goal\n (\"silently selecting a paid hosted provider\" must never happen by\n omission either).\n\n### Capability discovery\n\n`Registry.Names()` / `ByClass()` plus `Provider.Capability()` give a caller\n(the future `/image` routing bead) everything needed to ask \"what can I\noffer this user\" without a type switch on provider identity — this is the\nwhole point of the contract being provider-neutral.\n\n### Timeouts, size limits, cancellation\n\n- Every `Provider.Generate`/`Edit` call takes the caller's `ctx`; a\n provider that ignores cancellation is a provider bug, not a contract gap\n — mirrors every other Lifecycle-less call in the codebase (chat's\n `runtime.Chat`).\n- `ImageData` size limits are enforced by the *caller* (the future routing\n bead) before constructing a request, using a configured max — not by this\n contract, which has no config surface for \"current caller's limit\". This\n package only defines `ErrInputTooLarge` so a provider that discovers an\n oversized input itself (rather than trusting the caller) has a typed way\n to say so.\n\n## Testing\n\n- `internal/domain/image/{contract,registry}_test.go` mirror memory's test\n suite shape: registration (duplicate, nil provider), `Names`/`ByClass`\n filtering, `Get` miss, table-driven `Capability` checks via a fake\n provider.\n- `internal/infrastructure/configuration/validate_test.go` gains cases for\n `ImageConfig`: absent section, valid hosted+local, missing `Class`,\n missing credential, `Default` naming an unregistered/disabled provider.\n- No provider implementation exists yet, so no live-call tests — those\n belong to the hosted/local provider beads.\n\n## Hosted provider (#498)\n\n`github.com/samcharles93/ai-sdk` (already a repo dependency) has its own\nprovider-agnostic image facility: `image.Provider` (`Name`,\n`GenerateImage(ctx, GenerateImageRequest) (GenerateImageResponse, error)`),\n`image.Client`, and `registry.Registry.RegisterImage`/`.Image(name)` — the\nsame shape chat model registration already uses. Three backends implement it\n(azure, togetherai, xai); OpenAI does not.\n\n`internal/infrastructure/image.\u003cname\u003eProvider` wraps an `ai-sdk/image.Provider`\nbehind this package's `Provider` interface, the same way `curatorLLMRunner`\n(`internal/app/archied/main.go`) adapts `runtime.Runtime` for chat rather\nthan the domain layer talking to ai-sdk directly.\n\n**Decision: generation goes through ai-sdk, edit does not.** ai-sdk's\n`image.GenerateImageRequest` is text-to-image only — no input-image or mask\nfields — so there is nothing to adapt `EditRequest`/`Capability.Edit` onto.\nBlocking the entire hosted provider on an upstream ai-sdk change is not\nthis epic's call to make on someone else's project's timeline. Instead:\n\n- `Generate` wraps `ai-sdk/image.Client.GenerateImage` against whichever\n backend `ImageHostedProvider.Class` names (azure/togetherai/xai).\n- `Edit` is a direct HTTP client hitting the OpenAI-compatible\n `/images/edits` multipart endpoint, built the same way `minimax.Client`\n (`internal/tools/minimax/minimax.go`) is a direct client for an API\n ai-sdk doesn't cover — not a second abstraction layer, one `http.Client`\n call with request/response mapping into this contract's `EditRequest`/\n `Result`.\n- A provider that only wraps ai-sdk (no edit client configured) declares\n `Capability{Generate: true, Edit: false}` truthfully rather than\n returning `ErrUnsupported` at call time for something `Capability`\n already said it couldn't do.\n- If ai-sdk gains edit support upstream later, the direct client is deleted\n and `Edit` moves onto it — recorded here so that's a known, not a\n surprise, follow-up.\n\n**Provider error mapping** (every ai-sdk/HTTP error translated at the\nadapter boundary, never leaked as-is):\n\n| ai-sdk / HTTP condition | contract error |\n|---|---|\n| `image.ErrNoProvider`, `image.ErrAuthFailed`, missing/invalid credential, backend unreachable | `ErrUnavailable` |\n| `image.ErrEditNotSupported`, request needs a `Capability` the provider didn't declare | `ErrUnsupported` |\n| input image over configured byte limit (checked before the call, per \"Timeouts, size limits, cancellation\" above) | `ErrInputTooLarge` |\n| `ctx.Err() == context.DeadlineExceeded` | `ErrTimeout` |\n| `ctx.Err() == context.Canceled` | `ErrCanceled` |\n| `image.ErrRateLimited`, `image.ErrContentFiltered`, any other 4xx/5xx | wrapped verbatim under `ErrUnavailable` with the provider's own message as detail — a rate limit or content filter is \"unavailable for this request\", not a new sentinel the caller needs to branch on differently |\n\n**Testing**: `httptest.Server` fixtures for the edit client (request\nmultipart shape, response decoding, 4xx/5xx mapping); a fake `ai-sdk/image.Provider`\nfor the generate path so the adapter's translation is tested without a live\nbackend, matching #498's \"no live API key needed for tests\" criterion.\n\n## Local GPU provider (#499)\n\n**Decision: ComfyUI is the baseline recipe**, not a name this doc picked\narbitrarily — it is the de facto scriptable HTTP API for Stable-Diffusion-class\nlocal generation, already shaped like the async submit/poll pattern\n`minimax.Client.GenerateAndWait` established in this codebase (submit a\nworkflow, poll `/history/{id}` until the output node has files, download).\nNo other local backend is referenced anywhere in this repo or its docs, so\nthis decision has no competing prior art to reconcile — it is a fresh\nchoice, recorded here so the implementing bead does not re-litigate it.\n\n```go\n// internal/infrastructure/image/comfyui\ntype Config struct {\n Enabled bool\n BaseURL string // e.g. http://127.0.0.1:8188\n Workflow string // path to a saved ComfyUI workflow JSON template\n // (prompt/size/seed substituted at call time)\n Timeout time.Duration\n PollInterval time.Duration\n MaxWait time.Duration\n}\n```\n\n- `Generate`/`Edit` load `Workflow`, substitute the templated fields (prompt,\n size, seed, and for `Edit` the input image as a base64 node input — ComfyUI\n supports img2img/inpaint workflows natively, so `Capability.Edit`/`Mask`\n can both be true for a correctly authored workflow, unlike the hosted\n provider), POST to `/prompt`, then poll `/history/{prompt_id}` the same\n shape MiniMax's poll loop already proves out in this codebase.\n- `Capability.Sizes` is empty (ComfyUI workflows declare their own\n resolution in the template, not a fixed enum) — the contract already\n allows an empty `Sizes` to mean \"provider does not constrain size\".\n- Backend-down (`BaseURL` unreachable) and workflow-missing are both\n `ErrUnavailable`, not a startup failure — same \"degrade the capability,\n never break text chat\" rule the contract already states.\n- Per #499's own acceptance criteria, at least one recipe must be exercised\n end-to-end on real hardware or a reproducible harness, and no model\n weights are ever committed or fetched by Archie itself — the operator\n points `Workflow`/`BaseURL` at their own running ComfyUI instance and\n already-downloaded checkpoint, exactly like `[providers.ollama]` points at\n an already-running Ollama rather than Archie managing the model.\n\n**Testing**: a fake HTTP server standing in for ComfyUI's `/prompt` and\n`/history` endpoints (submit, poll, malformed history, timeout, job\nnot-found); a real-hardware smoke test documented as a manual `task`\ntarget per #503, not part of `task check` — the same reasoning `task vuln`\nand `task deadcode` are excluded from the gate.\n\n## `/image` routing and clarification flow (#500)\n\n**Call site: `internal/gateway/gateway.go`'s `Router`**, following the\n`/model` shape exactly — `/model`, `/spawn`, `/approve` etc. are already\n\"commands handled without the LLM\" (`Router.handleModel`, dispatched via\n`isLocalCommand`/`parseCmd` against `messaging.LocalCommandSpecs()`).\n`/image` is a new entry in `internal/domain/messaging/commands.go`'s\n`localCommandSpecs`, and a new `Router.handleImage(ctx, arg)` beside\n`handleModel`, gated on a new `Router.Images *image.Registry` field (nil =\n\"/image not configured\", the same pattern `Models`/`Tasks`/`Controller`\nalready use — not a new gating convention).\n\n**Decision: mechanism selection is per-request, not session-persisted.**\nThe epic allows \"a remembered preference only where existing session/config\nconventions support it\" — and checking `Router`'s actual fields\n(`ModelPersist`, `Identity`, the rest) shows there is no generic per-session\nkey/value store the neutral command path can lean on; `/model`'s own\n`--session`/`--global` scoping lives entirely in\n`internal/channels/telegram/model.go`, a Telegram-specific enrichment, not\nsomething `Router` provides generically. Inventing a new generic\nsession-preference store just for `/image` would be exactly the kind of\nscaffolding CLAUDE.md's Architectural Simplicity rule warns against for one\ncaller. So: every `/image` invocation with no explicit mechanism produces\nthe clarification prompt; an explicit `--hosted`/`--local` flag (mirroring\n`parseModelRest`'s `--provider` flag shape) skips it for that one call.\nRevisit if a second command wants session-scoped preference too — that's\nthe point at which shared infra earns its cost.\n\n**Flow**, matching `handleModel`'s reply-string shape (no separate state\nmachine object, since the whole exchange is \"ask once, use the answer\ninline in the next message\" — the model-selector's own worked example):\n\n1. `/image \u003cprompt\u003e` with no `--hosted`/`--local` → reply lists the\n registered classes (from `Registry.ByClass`) and asks the user to repeat\n the command with `--hosted` or `--local`. No provider is called.\n2. `/image --hosted \u003cprompt\u003e` or `--local` → resolves to\n `Registry.ByClass(class)`, picks `ImageConfig.Default` if it names a\n provider of that class, else the sole registered provider of that class,\n else (more than one, no default) asks which named provider.\n3. Edit requests (`/image --edit \u003cprompt\u003e` with an attached image on the\n inbound message) validate against `Capability.Edit`/`MaxInputs`/`Mask`\n before calling — an edit request to a generate-only provider is a\n clarification reply (\"this provider doesn't support editing\"), not a\n provider call that returns `ErrUnsupported`.\n4. A successful call returns a `Result`; delivery is the next section's\n concern, not this one's — `handleImage` returns the same\n `(string, error)` shape `handleModel` does for the text portion, and a\n separate mechanism (below) carries the actual image bytes.\n\n**Testing**: channel-neutral state-machine tests for `handleImage` against\na fake `*image.Registry` (mirrors `#500`'s own acceptance criterion); Telegram\nadapter tests only for wiring, not for re-testing the state machine.\n\n## Delivery through channels and web (#501)\n\nThis is the smallest section because most of the delivery path **already\nexists and is already proven** — by `generate_video`\n(`internal/tools/minimax`), which ships. Do not rebuild it.\n\nThe existing pipeline: a result carrying media returns\n`tools.MultimodalResult{IsMultimodal: true, URLs: []tools.MediaRef{{Type: \"image\", ...}}}`\n→ `multimodalMediaRefs` (`internal/app/archied/telegram_setup.go`) decodes\nit → `TurnStream.Media(gateway.MediaEvent)` → each channel's own\n`Media`/`MediaAttachment` handling. `MediaRef.Type` already documents\n`\"image\"` as a valid value alongside `\"video\"`; nothing in the pipeline is\nvideo-specific.\n\n**Per-channel state, checked directly rather than assumed:**\n\n| channel | image delivery | evidence |\n|---|---|---|\n| Telegram | **already works** | `internal/channels/telegram/media.go`'s `telegramMediaSender.dispatch` already branches on `att.Type` and calls `bot.SendPhoto` for `\"image\"` (`SendVideo` for `\"video\"` is the same function) |\n| Dashboard (webui) | **tracked gap, not new** | `internal/webui/api_chat.go`'s `chatStreamSink.Media` has no inline rendering path yet and degrades to a link/error string — already tracked as archie-core-1786748942243-6-f109697e. This epic does not need to fix it, but #501's acceptance criterion (\"unsupported paths report a clear limitation\") is already met by the existing degrade-to-text behavior; closing the dashboard gap is that bead's job, not this one's |\n| Email | **no attachment sender exists** | `grep MediaAttachment internal/channels/email` returns nothing. Out of scope for this epic per #496's non-goals (no channel-abstraction rework) — email delivery of a generated image degrades the same way the dashboard does which is acceptable under #501's \"or unsupported paths report a clear limitation\" clause |\n| Webhook | not applicable | webhook is an inbound intake channel (`internal/channels/webhook`), not an outbound chat reply target — no `/image` command can originate there in the first place |\n\n**The only new work `internal/tools/minimax` didn't already need**: the\nautonomous-tool call site. `/image` (previous section) calls a provider\ndirectly and does not go through the model's tool-calling loop at all.\nSeparately, exposing `generate_image`/`edit_image` as an ordinary tool\n(so an agent task can generate a diagram or illustration mid-task, not only\na human typing `/image`) is a thin wrapper — `internal/tools/image/tool.go`,\nsame shape as `internal/tools/minimax/tool.go`, calling\n`image.Registry.Get(name).Generate/Edit` and returning the identical\n`MultimodalResult` shape. Wired in `bootstrap.go` next to\n`registerMinimaxTool`, gated the same way (registered only when a provider\nresolved a working credential/backend, never advertised broken). This\nreuses the entire delivery pipeline above with zero new plumbing — the tool\ncall site is what's new, not the delivery.\n\n**Testing**: `multimodalMediaRefs` already has coverage for the shape;\nadd a table case for `Type: \"image\"` if one doesn't already exist. The tool\nwrapper gets the same fake-provider test shape as the routing flow above.\nNo new delivery-path tests are needed — that pipeline's tests belong to\nwhichever change originally proved it out.\n\n## Call site inventory\n\nEvery file a full implementation touches, so no bead re-derives this by\ngrep:\n\n| concern | file | change |\n|---|---|---|\n| domain contract | `internal/domain/image/{contract,registry}.go` | done (#497) |\n| config schema + validation | `internal/infrastructure/configuration/{types,validate}.go` | done (#497); extended here only if ComfyUI's `Workflow` field needs a new validation rule (path exists, non-empty when `Local.Enabled`) |\n| hosted provider | `internal/infrastructure/image/openai/*.go` (or per `Class`) | new (#498) |\n| local provider | `internal/infrastructure/image/comfyui/*.go` | new (#499) |\n| `/image` command spec | `internal/domain/messaging/commands.go` | new entry in `localCommandSpecs` |\n| `/image` routing | `internal/gateway/gateway.go` (`Router.Images`, `handleImage`) | new (#500) |\n| autonomous tool | `internal/tools/image/tool.go` | new (#501) |\n| tool registration | `internal/app/archied/bootstrap.go` (`registerImageTool`, beside `registerMinimaxTool`) | new (#501) |\n| `Registry` wiring | `internal/app/archied/bootstrap.go` (construct `image.Registry`, register configured providers, hand to `Router.Images`) | new — this is the \"nothing to wire until a real provider exists\" step the original contract doc deferred |\n| Telegram delivery | `internal/channels/telegram/media.go` | none — already handles `\"image\"` |\n| Dashboard delivery | `internal/webui/api_chat.go` | none in this epic — tracked separately (archie-core-1786748942243-6-f109697e) |\n| docs | `docs/` user-facing guide (#502) | new, against this design |\n\n## Execution: multi-agent team breakdown\n\nEach row is independently implementable from its own section above plus the\ncall-site inventory — an implementer does not need this whole document\nre-explained, only its row. Lenses are picked from\n`.claude/workflows/council.js` (`boundary`, `contract`, `deletionist`,\n`operator`, `maintainer`) by what each slice actually risks getting wrong,\nnot all five by default.\n\n| sub-feature | issue | implementer scope (call-site rows) | suggested lenses | why |\n|---|---|---|---|---|\n| Hosted provider | #498 | `internal/infrastructure/image/openai/*.go` (generate via ai-sdk, edit via direct HTTP client) | `lens-contract`, `lens-operator` | contract: the error-mapping table must actually produce the sentinels it claims, not leak ai-sdk/HTTP errors raw; operator: a live paid API's rate-limit/timeout/auth-failure paths are exactly the 3am-unattended-host case this lens exists for |\n| Local GPU provider | #499 | `internal/infrastructure/image/comfyui/*.go` (submit/poll against ComfyUI) | `lens-operator`, `lens-deletionist` | operator: backend-down, workflow-missing, and stuck-job failure modes on a host the operator's GPU actually runs; deletionist: a second, unevidenced \"recipe\" added later without real-hardware proof is exactly the speculative scaffolding this lens rejects |\n| `/image` routing | #500 | `internal/domain/messaging/commands.go`, `internal/gateway/gateway.go` (`Router.Images`, `handleImage`) | `lens-boundary`, `lens-maintainer` | boundary: a new `Router` field and its dependency on `image.Registry` must respect the same layering `Models`/`Tasks`/`Controller` already do; maintainer: this is the first local command with an explicit ask-once clarification exchange, not a single-shot reply — legibility for the next person who adds a command matters here |\n| Delivery + autonomous tool | #501 | `internal/tools/image/tool.go`, `internal/app/archied/bootstrap.go` (`registerImageTool`) | `lens-deletionist`, `lens-contract` | deletionist: the whole point of this section is \"reuse `generate_video`'s pipeline, do not rebuild it\" — this lens is the check that the implementation actually did that; contract: the new tool's `MultimodalResult`/`MediaRef{Type: \"image\"}` output must match the shape `multimodalMediaRefs` already decodes, not a near-miss |\n\nRun `/council --lenses \u003cpicked\u003e` once a row's implementation is gate-clean\n(`task check` passing), before opening its PR — same timing as any other\nopen-decision residue at the end of a session, not a standing per-commit\ngate.\n\n## File and link the beads\n\n`#498`–`#501` already exist (bd `archie-core-1786748942169-3-ae7cfaea`,\n`…-219-5-9844a72e`, `…-193-4-5072b564`, `…-243-6-f109697e`), filed under\nepic #496 before this document's decisions were made. Each one's\ndescription now links back to its matching section above instead of\nrestating it, and carries its row from the Execution table above so picking\nit up needs no round-trip back to this document.\n" }, { "url": "/dev/prds/inline-review/", @@ -274,7 +274,7 @@ "url": "/dev/prds/memory-engine-unification/", "title": "Memory engine unification: four scopes, CRUD, and one write path -- decision", "section": "PRDs", - "body": "# Memory engine unification: four scopes, CRUD, and one write path -- decision\n\n**Status:** Approved\n**Date:** 2026-09-16\n**Findings:** `memory-4` and its dependants in the adversarial sweep\n(`.local/issue-tracker`); the real defect is the split described below, for\nwhich `memory-4` is a symptom\n**Supersedes:** the draft `memory-turn-context.md` (its synchronous-read\nargument survives and is reused; its `Manager` method and consumer-owned port do\nnot)\n**Settles:** four open items in `docs/architecture/migration-decisions.md#5`\n(listed at the end)\n\n## Problem\n\nThere are **two live memory stores for one assistant**, split by which component\nwrites them, and the chat turn reads neither.\n\n| Family | Written by | Root | Format |\n|---|---|---|---|\n| `internal/memory` (`Manager`, `builtin.Provider`) | the chat agent's `memory_edit` tool via `memorytoolprovider` (`bootstrap.go`) | `\u003cworkDir\u003e/memory` | `MEMORY.md` / `USER.md` |\n| `internal/domain/memory` + `internal/infrastructure/memory` | the session curator (`sessioncurator.go`) | `\u003cworkDir\u003e/memory-engine` | identity-keyed blocks |\n\nThey share no code, no format and no scope vocabulary. Neither reaches a prompt:\n`Manager.SystemPromptBlock` (`internal/memory/manager.go`) has no production\ncaller anywhere, so the agent can write memory through a tool and never read it\nback.\n\nThree further defects fall out of the same split:\n\n1. **The engine's contract is not CRUD.** `Store` is `Write`/`Query`/`List`/\n `Forget` — create, read, delete, and **no Update**. The retired file provider\n *does* have `handleAdd`/`handleReplace`/`handleRemove`, so the family being\n deleted is operationally richer than its replacement.\n2. **There is no scope model.** The contract carries one opaque `Identity\n string`, and the curator fills it with a **session id** (`sessioncurator.go`)\n — which is none of the four scopes `docs/architecture/agent-system.md`\n requires, and stops meaning anything when the session ends.\n3. **The chat turn has no user identity on half the channels.** Telegram and\n email populate `SenderID` with a person; the dashboard populates it with\n nothing, and a webhook populates it with nothing either — a route path names\n a source, not a person, so `webhook.go` leaves `SenderID` empty and carries\n the route in `Inbound.BudgetKey` for rate limiting. `sender_id` is persisted\n (`session_store_sqlite.go`), so whatever a channel puts there reaches the\n consumers that read it.\n\n`agent-system.md` says the write-scope selection and cross-scope sharing\nrules \"require a focused memory design decision\". This is that decision.\n\n## Decision\n\n### 1. One engine, addressed by four typed scopes\n\n`internal/domain/memory` is the only memory engine. `Observation.Identity`,\n`Query.Identity`, `Record.Identity` and `List(identity)` are deleted, not\nextended: a field whose meaning depends on the caller cannot carry a scope.\n\n```go\ntype ScopeKind string\n\nconst (\n ScopeGlobal ScopeKind = \"global\" // shared across the application\n ScopeAgent ScopeKind = \"agent\" // private to one Agent\n ScopeUser ScopeKind = \"user\" // one user, across their Agents\n ScopeAgentUser ScopeKind = \"agent-user\" // one Agent's relationship with one user\n)\n\ntype Scope struct {\n Kind ScopeKind\n Agent AgentID // required for ScopeAgent, ScopeAgentUser\n User IdentityID // required for ScopeUser, ScopeAgentUser\n}\n\nfunc (s Scope) Validate() error // rejects a component present where it is not required\nfunc (s Scope) Key() string // canonical, collision-free storage key\n```\n\n`AgentID` and `IdentityID` are opaque. This package never resolves or\ncanonicalises them.\n\nA turn's resolved identity is one value:\n\n```go\ntype Subject struct {\n AgentID AgentID\n UserID IdentityID // \"\" when the channel carries no per-person identity\n}\n\nfunc (s Subject) Scopes() []Scope // readable, most specific first\nfunc (s Subject) WritableScopes() []Scope // never includes ScopeGlobal\n```\n\n**Access control is naming a scope, and nothing else.** The read path names\nexactly the scopes of the resolved `Subject`; the engine never unions beyond what\nit is handed and applies no policy of its own. An engine that enforced policy\nwould need to know about channels, sessions and bindings, and every backend would\nreimplement it. \"Agent-wide is private to its owning Agent by default\"\n(`agent-system.md`) falls out mechanically, because `ScopeAgent{a}` and\n`ScopeAgent{b}` are different storage keys.\n\n`ScopeGlobal` is never model-writable. Writing it is an operator action; nothing\nproduces one yet, but the contract models it because the read set includes it.\n\n### 2. CRUD, with revisions retained\n\n```go\ntype Store interface {\n Create(ctx context.Context, in NewRecord) (Record, error)\n Get(ctx context.Context, scope Scope, id RecordID) (Record, error) // ErrNotFound\n Query(ctx context.Context, q Query) ([]Record, error) // heads only\n List(ctx context.Context, scope Scope) ([]Record, error) // heads only\n Update(ctx context.Context, in RecordUpdate) (Record, error) // ErrNotFound, ErrStaleRevision\n Forget(ctx context.Context, scope Scope, id RecordID) error // idempotent\n Revisions(ctx context.Context, scope Scope, id RecordID) ([]Revision, error)\n}\n```\n\n`Get` and `Forget` take a scope. The current `Forget` scans every identity's\nstore to find a marker (`builtin_engine.go`) purely because the contract\ngave it nothing better.\n\n`Record` carries what `agent-system.md` requires be retained: `Scope`,\n`Kind`, `Content`, `Revision`, `Author`, `OriginUser`, `Source`, `CreatedAt`,\n`UpdatedAt`. `Metadata` is dropped — nothing sets it and the parser never\nrestores it, so it is a field that silently does not round-trip.\n\n`Update` supersedes rather than overwrites. The superseded state is appended to\nthe scope's `HISTORY.md` **before** the live block is replaced, so a crash\nbetween the two leaves an extra history entry and an unchanged record —\nrecoverable — never a record whose previous state was lost. `Update` carries no\n`Scope`: a record may never move between scopes, and the API makes that\nunrepresentable rather than validating it. `Expected` optionally names the\nrevision being replaced and returns `ErrStaleRevision` on mismatch, which matters\nbecause two processes (`archied` and `archie-gateway`) can hold the same scope's\nfile.\n\n`Forget` records a deleted revision before removing the live block, so a record's\nprovenance outlives its content.\n\n### 3. The caller resolves identity; the engine never does\n\nResolution happens in the gateway turn path, where the native sender id and the\nsession are both in hand, and it is supplied **per channel by the composition**:\n\n```go\n// TurnRunnerConfig\n// UserIdentity resolves the initiating user's identity for one inbound message.\n// false means the channel carries no per-person identity, and the turn gets\n// global and agent scopes only. Nil means the same.\nUserIdentity func(msg messaging.Message) (IdentityID, bool)\n```\n\nThis must be injectable rather than a blanket `channel + \":\" + SenderID`\nconcatenation, because a channel's sender id is not necessarily a person: a\nwebhook's route path names a source, and treating it as an identity would give\na URL one -- which is why `webhook.go` leaves `SenderID` empty and carries the\nroute in `Inbound.BudgetKey` instead. Telegram and email supply their sender\nid; the dashboard and webhook resolve nothing.\n\n**Fail closed.** When `UserID` is empty the turn proceeds with global and agent\nscopes only, and a write targeting `user` or `agent-user` fails with a clear\nerror. `agent-system.md` forbids showing one user's memory to another, so\n\"we do not know who this is\" must mean \"show none of it\", never \"fall back to a\nwider scope\".\n\n### 4. The chat read path\n\nSynchronous and total. It runs inside `prepareTurn` (`turn.go`) before the\nprompt is built, never returns an error to the caller, and degrades to an empty\nblock on any engine failure or provider panic. A fire-and-forget hook would race\nthe prompt build in the same turn, which is why this is not a lifecycle hook.\n\nThe read is a scope-only `Query` over the subject's scopes — **not** ranked\nsearch, which remains open in `migration-decisions.md#5`. A ranked read on the\nhot path with no ranking implementation would be a lie.\n\n`SystemPromptConfig` gains a `Memory` field, rendered into\n`templates/archie.md.tpl` between the `\u003ctools\u003e` and `\u003cenv\u003e` blocks:\n\n```\n\u003cmemory purpose=\"durable_context\" trust=\"data\"\u003e\n```\n\n`trust=\"data\"` and inclusion in the precedence clause follow the `\u003ctools\u003e`/`\u003cenv\u003e`\nprecedent exactly: memory content is model-written from untrusted input and must\nbe framed as data, never instructions. The rendered block carries each record's\nshort id, because the write path addresses records by id — one source of truth\nfor both directions.\n\nBoth a per-scope record limit and a byte cap on the rendered block are named\nconstants. The cap is about size, not latency; a local read needs no timeout.\n\n**Prerequisite:** `setupGateways` runs before `setupMemoryAll`\n(`main.go` vs `:428`), and `startGatewayRuntime` before `setupMemory`\n(`gateway.go` vs `:115`), so the engine is nil at every turn-runner\nconstruction site. The ordering is the root-cause fix and nothing in either\nmemory step reads what those later steps produce.\n\n### 5. The chat write path\n\nThe memory tool is built **per turn**, not registered once at boot, because it\nmust write into a scope derived from the turn's resolved subject:\n\n| action | engine call |\n|---|---|\n| `create` | `Create` |\n| `update` | `Update` (by id) |\n| `delete` | `Forget` (by id) |\n| `list` | `List` |\n\n**The model chooses the scope kind; it never chooses the ids.** Agent and user\nids come from the turn's resolution and there is no schema field to put anything\nelse in — even a model that emits another user's id has nowhere to write it. That\nis the root-cause form of `agent-system.md`, matching this system's other\nnon-waivable boundaries (the model never runs git; gates cannot be waived by the\nagent). `global` is not offered at all.\n\nThe tool takes a record **id** where the retired provider took a content\nsubstring (`handleReplace`/`handleRemove`). The substring addressing existed\nbecause a markdown file was hand-edited and the model had no stable handle on a\nblock; with ids rendered into the prompt it disappears, and one addressing scheme\nreplaces two.\n\n### 6. The curator's write\n\n`sessioncurator.reviewOne` stops writing `Identity: sessionID` and writes\n`ScopeAgentUser{AgentID: \u003csession's agent\u003e, UserID: \u003cparticipant\u003e}`. Relationship\nscope is the only defensible home: agent-wide is explicitly unsafe for user facts\nand would leak user A's extracted facts to user B through the same agent;\nuser-wide would pool facts across agents; and a session id cannot survive the\nsession ending, which is the whole requirement.\n\nThis needs data the tree does not persist: the SQLite `messages` table\ngains `sender_id`, and `curator.SessionSummary` / `ConversationMessage` gain the\nagent and participant. The participant is derived from the session's own\nuser-role messages: exactly one distinct non-empty sender writes; zero (a\ndashboard or webhook session) or several (a group chat) writes nothing and\nrecords why. Ambiguity fails closed rather than guessing.\n\nThe payoff is the point of the exercise: chat-written and curator-derived memory\nnow occupy the same scopes, so a future turn's prompt contains both, and a\ncurator's bad note becomes id-addressable and therefore correctable by the agent.\n\n### 7. The content scanner moves to the engine\n\n`internal/memory/scanner.go` and its installation at `bootstrap.go` are what\nmake `configuration.md`'s \"Memory safety scanner\" row describe a control that\nactually runs. Deleting the package would silently remove prompt-injection\nscanning on memory writes. The scanner moves into the engine and is applied in\n`Create` and `Update`, which is the single choke point every producer crosses —\nso the curator's model-extracted content is covered too, not just the chat tool's.\n\n### 8. `internal/memory` is deleted\n\nNot shimmed, not partially, not behind a flag: `manager.go`, `provider.go`,\n`lifecycle.go`, `sync.go`, `scanner.go` (moved), `schema.go`, `backup.go`,\n`config.go`, `builtin/provider.go`, and their tests. Its markdown store\n(`builtin/store.go`) is the one survivor, relocated to\n`internal/infrastructure/memory/builtin/` — `organisation.md`'s \"a package owns\nits on-disk format end-to-end\" makes infrastructure its correct home, and the\nengine already parses that format.\n\nAlso deleted: `internal/tools/provider/memory`'s manager wrapping (rewired to the\nengine), `config.Memory.{Provider,ProviderConfig}` and the\n`validateMemory` rejection that exists only to complain about `Provider`\n(`validate.go`), and `webui.MemoryStatus`/`MemoryProviderHandle` — never\nimplemented in production, so `/api/memory` reports empty forever unless it is\nrepointed at the engine registry.\n\nExisting data at `\u003cworkDir\u003e/memory` is left on disk, unreferenced. No migration.\n`\u003cworkDir\u003e/memory-engine` is likewise orphaned: its directories were keyed by\n`sha256(sessionID)` and no scope key can hash to one.\n\n## What we deliberately do NOT do\n\n- **No cross-scope sharing or grants in this wave.** Promoting a record between\n scopes is a future explicit command, and `Update` cannot change scope, so it\n cannot happen by accident.\n- **No canonical user identity.** `SenderID` is channel-native, so one person on\n Telegram and email is two identities. Unifying needs an Identity aggregate that\n does not exist (`internal/domain` has no identity package and `IdentityID`\n appears nowhere in Go). The contract does not change when it lands.\n- **No ranked retrieval.** `Query.Text` stays, unused by the prompt path.\n- **No data migration**, per the decision above.\n- **No prompt-memory timeout mechanism.** The read is bounded by construction.\n- **No `Metadata` field.** It is write-only; a caller needing it later\n needs one field, not a speculative contract.\n\n## Acceptance criteria\n\n1. A record created through the memory tool in turn 1 appears in turn 2's prompt,\n asserted on content the test planted, with its id.\n2. The same record is **absent** from a different user's prompt on the same\n channel and agent. Isolation asserted end to end, not at the engine boundary.\n3. An agent-scope record is visible to both of that agent's users; a global\n record is visible to a second agent; an agent-user record is not.\n4. A curator pass over a session writes a record a later turn reads, and which\n survives that session being deleted.\n5. `create` → prompt shows it; `update` → prompt shows new content and\n `Revisions` returns the prior content with author and time; `delete` → gone\n from prompt and `List`. Round-trip is asserted as `parse(render(r)) == r`,\n never against a fixture string.\n6. A panicking engine and a failing engine both complete the turn with no memory\n block.\n7. A channel with no resolvable user renders only global and agent scopes, and\n the tool refuses `user`/`agent-user` with a reason. A webhook route path never\n becomes a user identity.\n8. A write that would exceed the file cap fails loudly rather than silently\n dropping content.\n9. `internal/memory` is gone, `task check` is green, and no document describes a\n manager that no longer exists.\n\nEvery criterion asserts behaviour or planted content. None asserts on the text of\na hand-written file, and none pins that a method was called. In particular none\nasserts the composition calls setup in a given order: if the ordering were\nswapped back, criteria 1 and 4 would fail, and that is the assertion.\n\n## Sequencing\n\nEach step is independently verifiable and lands green on its own.\n\n1. **Persist the participant.** `sender_id` column on `messages`,\n insert/select/migration, `curator.SessionSummary.AgentID` and\n `ConversationMessage.SenderID`. Touches no memory contract and is shippable\n alone — nothing consumes it yet.\n2. **Scope, CRUD and revisions.** The `internal/domain/memory` contract, the\n builtin engine with `HISTORY.md`, the store relocation, the scanner move, and\n the curator rewritten onto agent-user scope (it is the engine's only live\n consumer, so it belongs in this slice). *Depends on 1 for the curator half.*\n3. **Chat read path.** The `Memory` seam, the template block, subject resolution,\n and the composition-ordering fix. *Depends on 2.*\n4. **Chat write path.** The tool adapter onto CRUD and the turn's subject on the\n context. *Depends on 3, because a write is only observable through the read.*\n5. **Delete `internal/memory` and the dead config/dashboard surface**, and\n rewrite `ARCHITECTURE.md`'s \"Memory engine family\" section to describe the\n engine family rather than a manager it deleted. *Depends on 3 and 4.*\n\nSteps 3 and 4 are separable from each other; step 1 is separable from everything.\n\n## Settles these open items in `migration-decisions.md` §5\n\n| §5 open item | Settled by |\n|---|---|\n| authoritative memory record and revision model | §1, §2 — the live marker becomes JSON, since the ratified `\u003c!--mem:\u003culid\u003e--\u003e` cannot carry provenance; superseded states live in a per-scope `HISTORY.md` |\n| provider and infrastructure boundaries | §2 — contract in `internal/domain/memory`, builtin format in `internal/infrastructure/memory/builtin` |\n| retrieval and access enforcement | §1 — caller-named scope sets, no engine-side policy |\n| provenance representation | §2 |\n\n## Not determined\n\n- **Group chats produce no derived memory**, because the curator fails closed on\n mixed-sender sessions. Whether extraction should instead become\n per-participant is a product call.\n- **Two processes, one scope file.** After step 3 both `archied` and\n `archie-gateway` hold an engine over the same root, and the engine caches a\n store per scope and rewrites the whole file, so concurrent writers on one scope\n can lose a block. `Update.Expected` makes that detectable for one record; it\n does not make the file safe. A reload-before-mutate or advisory lock is the fix\n and is not designed here.\n- **Unbounded history.** `HISTORY.md` grows without limit; no compaction policy.\n- **Size bounds.** `defaultMaxFileBytes` is now per scope rather than per\n identity; whether that is right is unexamined.\n- **Whether the dashboard should carry a user identity.** It has one bearer\n token, not users. Until it has users, web turns are correctly limited to global\n and agent scope.\n" + "body": "# Memory engine unification: four scopes, CRUD, and one write path -- decision\n\n**Status:** Approved\n**Date:** 2026-09-16\n**Findings:** `memory-4` and its dependants in the adversarial sweep\n(`.local/issue-tracker`); the real defect is the split described below, for\nwhich `memory-4` is a symptom\n**Supersedes:** the draft `memory-turn-context.md` (its synchronous-read\nargument survives and is reused; its `Manager` method and consumer-owned port do\nnot)\n**Settles:** four open items in `docs/architecture/migration-decisions.md#5`\n(listed at the end)\n\n## Problem\n\nThere are **two live memory stores for one assistant**, split by which component\nwrites them, and the chat turn reads neither.\n\n| Family | Written by | Root | Format |\n|---|---|---|---|\n| `internal/memory` (`Manager`, `builtin.Provider`) | the chat agent's `memory_edit` tool via `memorytoolprovider` (`bootstrap.go`) | `\u003cworkDir\u003e/memory` | `MEMORY.md` / `USER.md` |\n| `internal/domain/memory` + `internal/infrastructure/memory` | the session curator (`sessioncurator.go`) | `\u003cworkDir\u003e/memory-engine` | identity-keyed blocks |\n\nThey share no code, no format and no scope vocabulary. Neither reaches a prompt:\n`Manager.SystemPromptBlock` (`internal/memory/manager.go`) has no production\ncaller anywhere, so the agent can write memory through a tool and never read it\nback.\n\nThree further defects fall out of the same split:\n\n1. **The engine's contract is not CRUD.** `Store` is `Write`/`Query`/`List`/\n `Forget` — create, read, delete, and **no Update**. The retired file provider\n *does* have `handleAdd`/`handleReplace`/`handleRemove`, so the family being\n deleted is operationally richer than its replacement.\n2. **There is no scope model.** The contract carries one opaque `Identity\n string`, and the curator fills it with a **session id** (`sessioncurator.go`)\n — which is none of the four scopes `docs/architecture/agent-system.md`\n requires, and stops meaning anything when the session ends.\n3. **The chat turn has no user identity on half the channels.** Telegram and\n email populate `SenderID` with a person; the dashboard populates it with\n nothing, and a webhook populates it with nothing either — a route path names\n a source, not a person, so `webhook.go` leaves `SenderID` empty and carries\n the route in `Inbound.BudgetKey` for rate limiting. `sender_id` is persisted\n (`session_store_sqlite.go`), so whatever a channel puts there reaches the\n consumers that read it.\n\n`agent-system.md` says the write-scope selection and cross-scope sharing\nrules \"require a focused memory design decision\". This is that decision.\n\n## Decision\n\n### 1. One engine, addressed by four typed scopes\n\n`internal/domain/memory` is the only memory engine. `Observation.Identity`,\n`Query.Identity`, `Record.Identity` and `List(identity)` are deleted, not\nextended: a field whose meaning depends on the caller cannot carry a scope.\n\n```go\ntype ScopeKind string\n\nconst (\n ScopeGlobal ScopeKind = \"global\" // shared across the application\n ScopeAgent ScopeKind = \"agent\" // private to one Agent\n ScopeUser ScopeKind = \"user\" // one user, across their Agents\n ScopeAgentUser ScopeKind = \"agent-user\" // one Agent's relationship with one user\n)\n\ntype Scope struct {\n Kind ScopeKind\n Agent AgentID // required for ScopeAgent, ScopeAgentUser\n User IdentityID // required for ScopeUser, ScopeAgentUser\n}\n\nfunc (s Scope) Validate() error // rejects a component present where it is not required\nfunc (s Scope) Key() string // canonical, collision-free storage key\n```\n\n`AgentID` and `IdentityID` are opaque. This package never resolves or\ncanonicalises them.\n\nA turn's resolved identity is one value:\n\n```go\ntype Subject struct {\n AgentID AgentID\n UserID IdentityID // \"\" when the channel carries no per-person identity\n}\n\nfunc (s Subject) Scopes() []Scope // readable, most specific first\nfunc (s Subject) WritableScopes() []Scope // never includes ScopeGlobal\n```\n\n**Access control is naming a scope, and nothing else.** The read path names\nexactly the scopes of the resolved `Subject`; the engine never unions beyond what\nit is handed and applies no policy of its own. An engine that enforced policy\nwould need to know about channels, sessions and bindings, and every backend would\nreimplement it. \"Agent-wide is private to its owning Agent by default\"\n(`agent-system.md`) falls out mechanically, because `ScopeAgent{a}` and\n`ScopeAgent{b}` are different storage keys.\n\n`ScopeGlobal` is never model-writable. Writing it is an operator action; nothing\nproduces one yet, but the contract models it because the read set includes it.\n\n### 2. CRUD, with revisions retained\n\n```go\ntype Store interface {\n Create(ctx context.Context, in NewRecord) (Record, error)\n Get(ctx context.Context, scope Scope, id RecordID) (Record, error) // ErrNotFound\n Query(ctx context.Context, q Query) ([]Record, error) // heads only\n List(ctx context.Context, scope Scope) ([]Record, error) // heads only\n Update(ctx context.Context, in RecordUpdate) (Record, error) // ErrNotFound, ErrStaleRevision\n Forget(ctx context.Context, scope Scope, id RecordID) error // idempotent\n Revisions(ctx context.Context, scope Scope, id RecordID) ([]Revision, error)\n}\n```\n\n`Get` and `Forget` take a scope. The current `Forget` scans every identity's\nstore to find a marker (`builtin_engine.go`) purely because the contract\ngave it nothing better.\n\n`Record` carries what `agent-system.md` requires be retained: `Scope`,\n`Kind`, `Content`, `Revision`, `Author`, `OriginUser`, `Source`, `CreatedAt`,\n`UpdatedAt`. `Metadata` is dropped — nothing sets it and the parser never\nrestores it, so it is a field that silently does not round-trip.\n\n`Update` supersedes rather than overwrites. The superseded state is appended to\nthe scope's `HISTORY.md` **before** the live block is replaced, so a crash\nbetween the two leaves an extra history entry and an unchanged record —\nrecoverable — never a record whose previous state was lost. `Update` carries no\n`Scope`: a record may never move between scopes, and the API makes that\nunrepresentable rather than validating it. `Expected` optionally names the\nrevision being replaced and returns `ErrStaleRevision` on mismatch, which matters\nbecause two processes (`archied` and `archie-gateway`) can hold the same scope's\nfile.\n\n`Forget` records a deleted revision before removing the live block, so a record's\nprovenance outlives its content.\n\n### 3. The caller resolves identity; the engine never does\n\nResolution happens in the gateway turn path, where the native sender id and the\nsession are both in hand, and it is supplied **per channel by the composition**:\n\n```go\n// TurnRunnerConfig\n// UserIdentity resolves the initiating user's identity for one inbound message.\n// false means the channel carries no per-person identity, and the turn gets\n// global and agent scopes only. Nil means the same.\nUserIdentity func(msg messaging.Message) (IdentityID, bool)\n```\n\nThis must be injectable rather than a blanket `channel + \":\" + SenderID`\nconcatenation, because a channel's sender id is not necessarily a person: a\nwebhook's route path names a source, and treating it as an identity would give\na URL one -- which is why `webhook.go` leaves `SenderID` empty and carries the\nroute in `Inbound.BudgetKey` instead. Telegram and email supply their sender\nid; the dashboard and webhook resolve nothing.\n\n**Fail closed.** When `UserID` is empty the turn proceeds with global and agent\nscopes only, and a write targeting `user` or `agent-user` fails with a clear\nerror. `agent-system.md` forbids showing one user's memory to another, so\n\"we do not know who this is\" must mean \"show none of it\", never \"fall back to a\nwider scope\".\n\n### 4. The chat read path\n\nSynchronous and total. It runs inside `prepareTurn` (`turn.go`) before the\nprompt is built, never returns an error to the caller, and degrades to an empty\nblock on any engine failure or provider panic. A fire-and-forget hook would race\nthe prompt build in the same turn, which is why this is not a lifecycle hook.\n\nThe read is a scope-only `Query` over the subject's scopes — **not** ranked\nsearch, which remains open in `migration-decisions.md#5`. A ranked read on the\nhot path with no ranking implementation would be a lie.\n\n`SystemPromptConfig` gains a `Memory` field, rendered into\n`templates/archie.md.tpl` between the `\u003ctools\u003e` and `\u003cenv\u003e` blocks:\n\n```\n\u003cmemory purpose=\"durable_context\" trust=\"data\"\u003e\n```\n\n`trust=\"data\"` and inclusion in the precedence clause follow the `\u003ctools\u003e`/`\u003cenv\u003e`\nprecedent exactly: memory content is model-written from untrusted input and must\nbe framed as data, never instructions. The rendered block carries each record's\nshort id, because the write path addresses records by id — one source of truth\nfor both directions.\n\nBoth a per-scope record limit and a byte cap on the rendered block are named\nconstants. The cap is about size, not latency; a local read needs no timeout.\n\n**Prerequisite:** `setupGateways` runs before `setupMemoryAll`\n(`main.go` vs `:428`), and `startGatewayRuntime` before `setupMemory`\n(`gateway.go` vs `:115`), so the engine is nil at every turn-runner\nconstruction site. The ordering is the root-cause fix and nothing in either\nmemory step reads what those later steps produce.\n\n### 5. The chat write path\n\nThe memory tool is built **per turn**, not registered once at boot, because it\nmust write into a scope derived from the turn's resolved subject:\n\n| action | engine call |\n|---|---|\n| `create` | `Create` |\n| `update` | `Update` (by id) |\n| `delete` | `Forget` (by id) |\n| `list` | `List` |\n\n**The model chooses the scope kind; it never chooses the ids.** Agent and user\nids come from the turn's resolution and there is no schema field to put anything\nelse in — even a model that emits another user's id has nowhere to write it. That\nis the root-cause form of `agent-system.md`, matching this system's other\nnon-waivable boundaries (the model never runs git; gates cannot be waived by the\nagent). `global` is not offered at all.\n\nThe tool takes a record **id** where the retired provider took a content\nsubstring (`handleReplace`/`handleRemove`). The substring addressing existed\nbecause a markdown file was hand-edited and the model had no stable handle on a\nblock; with ids rendered into the prompt it disappears, and one addressing scheme\nreplaces two.\n\n### 6. The curator's write\n\n`sessioncurator.reviewOne` stops writing `Identity: sessionID` and writes\n`ScopeAgentUser{AgentID: \u003csession's agent\u003e, UserID: \u003cparticipant\u003e}`. Relationship\nscope is the only defensible home: agent-wide is explicitly unsafe for user facts\nand would leak user A's extracted facts to user B through the same agent;\nuser-wide would pool facts across agents; and a session id cannot survive the\nsession ending, which is the whole requirement.\n\nThis needs data the tree does not persist: the SQLite `messages` table\ngains `sender_id`, and `curator.SessionSummary` / `ConversationMessage` gain the\nagent and participant. The participant is derived from the session's own\nuser-role messages: exactly one distinct non-empty sender writes; zero (a\ndashboard or webhook session) or several (a group chat) writes nothing and\nrecords why. Ambiguity fails closed rather than guessing.\n\nThe payoff is the point of the exercise: chat-written and curator-derived memory\nnow occupy the same scopes, so a future turn's prompt contains both, and a\ncurator's bad note becomes id-addressable and therefore correctable by the agent.\n\n### 7. The content scanner moves to the engine\n\n`internal/memory/scanner.go` and its installation at `bootstrap.go` are what\nmake `configuration.md`'s \"Memory safety scanner\" row describe a control that\nactually runs. Deleting the package would silently remove prompt-injection\nscanning on memory writes. The scanner moves into the engine and is applied in\n`Create` and `Update`, which is the single choke point every producer crosses —\nso the curator's model-extracted content is covered too, not just the chat tool's.\n\n### 8. `internal/memory` is deleted\n\nNot shimmed, not partially, not behind a flag: `manager.go`, `provider.go`,\n`lifecycle.go`, `sync.go`, `scanner.go` (moved), `schema.go`, `backup.go`,\n`config.go`, `builtin/provider.go`, and their tests. Its markdown store\n(`builtin/store.go`) is the one survivor, relocated to\n`internal/infrastructure/memory/builtin/` — `organisation.md`'s \"a package owns\nits on-disk format end-to-end\" makes infrastructure its correct home, and the\nengine already parses that format.\n\nAlso deleted: `internal/tools/provider/memory`'s manager wrapping (rewired to the\nengine), `config.Memory.{Provider,ProviderConfig}` and the\n`validateMemory` rejection that exists only to complain about `Provider`\n(`validate.go`), and `webui.MemoryStatus`/`MemoryProviderHandle` — never\nimplemented in production, so `/api/memory` reports empty forever unless it is\nrepointed at the engine registry.\n\nExisting data at `\u003cworkDir\u003e/memory` is left on disk, unreferenced. No migration.\n`\u003cworkDir\u003e/memory-engine` is likewise orphaned: its directories were keyed by\n`sha256(sessionID)` and no scope key can hash to one.\n\n## What we deliberately do NOT do\n\n- **No cross-scope sharing or grants in this wave.** Promoting a record between\n scopes is a future explicit command, and `Update` cannot change scope, so it\n cannot happen by accident.\n- **No canonical user identity.** `SenderID` is channel-native, so one person on\n Telegram and email is two identities. Unifying needs an Identity aggregate that\n does not exist (`internal/domain` has no identity package and `IdentityID`\n appears nowhere in Go). The contract does not change when it lands.\n- **No ranked retrieval.** `Query.Text` stays, unused by the prompt path.\n- **No data migration**, per the decision above.\n- **No prompt-memory timeout mechanism.** The read is bounded by construction.\n- **No `Metadata` field.** It is write-only; a caller needing it later\n needs one field, not a speculative contract.\n\n## Acceptance criteria\n\n1. A record created through the memory tool in turn 1 appears in turn 2's prompt,\n asserted on content the test planted, with its id.\n2. The same record is **absent** from a different user's prompt on the same\n channel and agent. Isolation asserted end to end, not at the engine boundary.\n3. An agent-scope record is visible to both of that agent's users; a global\n record is visible to a second agent; an agent-user record is not.\n4. A curator pass over a session writes a record a later turn reads, and which\n survives that session being deleted.\n5. `create` → prompt shows it; `update` → prompt shows new content and\n `Revisions` returns the prior content with author and time; `delete` → gone\n from prompt and `List`. Round-trip is asserted as `parse(render(r)) == r`,\n never against a fixture string.\n6. A panicking engine and a failing engine both complete the turn with no memory\n block.\n7. A channel with no resolvable user renders only global and agent scopes, and\n the tool refuses `user`/`agent-user` with a reason. A webhook route path never\n becomes a user identity.\n8. A write that would exceed the file cap fails loudly rather than silently\n dropping content.\n9. `internal/memory` is gone, `task check` is green, and no document describes a\n manager that no longer exists.\n\nEvery criterion asserts behaviour or planted content. None asserts on the text of\na hand-written file, and none pins that a method was called. In particular none\nasserts the composition calls setup in a given order: if the ordering were\nswapped back, criteria 1 and 4 would fail, and that is the assertion.\n\n## Sequencing\n\nEach step is independently verifiable and lands green on its own.\n\n1. **Persist the participant.** `sender_id` column on `messages`,\n insert/select/migration, `curator.SessionSummary.AgentID` and\n `ConversationMessage.SenderID`. Touches no memory contract and is shippable\n alone — nothing consumes it yet.\n2. **Scope, CRUD and revisions.** The `internal/domain/memory` contract, the\n builtin engine with `HISTORY.md`, the store relocation, the scanner move, and\n the curator rewritten onto agent-user scope (it is the engine's only live\n consumer, so it belongs in this slice). *Depends on 1 for the curator half.*\n3. **Chat read path.** The `Memory` seam, the template block, subject resolution,\n and the composition-ordering fix. *Depends on 2.*\n4. **Chat write path.** The tool adapter onto CRUD and the turn's subject on the\n context. *Depends on 3, because a write is only observable through the read.*\n5. **Delete `internal/memory` and the dead config/dashboard surface**, and\n rewrite the \"Memory engine family\" section of\n `docs/architecture/plugins-and-extensions.md` to describe the\n engine family rather than a manager it deleted. *Depends on 3 and 4.*\n\nSteps 3 and 4 are separable from each other; step 1 is separable from everything.\n\n## Settles these open items in `migration-decisions.md` §5\n\n| §5 open item | Settled by |\n|---|---|\n| authoritative memory record and revision model | §1, §2 — the live marker becomes JSON, since the ratified `\u003c!--mem:\u003culid\u003e--\u003e` cannot carry provenance; superseded states live in a per-scope `HISTORY.md` |\n| provider and infrastructure boundaries | §2 — contract in `internal/domain/memory`, builtin format in `internal/infrastructure/memory/builtin` |\n| retrieval and access enforcement | §1 — caller-named scope sets, no engine-side policy |\n| provenance representation | §2 |\n\n## Not determined\n\n- **Group chats produce no derived memory**, because the curator fails closed on\n mixed-sender sessions. Whether extraction should instead become\n per-participant is a product call.\n- **Two processes, one scope file.** After step 3 both `archied` and\n `archie-gateway` hold an engine over the same root, and the engine caches a\n store per scope and rewrites the whole file, so concurrent writers on one scope\n can lose a block. `Update.Expected` makes that detectable for one record; it\n does not make the file safe. A reload-before-mutate or advisory lock is the fix\n and is not designed here.\n- **Unbounded history.** `HISTORY.md` grows without limit; no compaction policy.\n- **Size bounds.** `defaultMaxFileBytes` is now per scope rather than per\n identity; whether that is right is unexamined.\n- **Whether the dashboard should carry a user identity.** It has one bearer\n token, not users. Until it has users, web turns are correctly limited to global\n and agent scope.\n" }, { "url": "/dev/prds/messaging-service-boundary/", @@ -364,7 +364,7 @@ "url": "/dev/prds/service-decomposition/", "title": "Service decomposition -- decision", "section": "PRDs", - "body": "# Service decomposition -- decision\n\n**Status:** Draft\n**Date:** 2026-09-05\n**Beads epic:** archie-core-8cda\n**Prior art:** `docs/inspiration/2026-09-05-awx-service-decomposition-and-ansible-task-model.md`,\n`docs/inspiration/module-reference-awx-decomposition-vs-archie.md`,\n`docs/inspiration/service-discovery-research-2026-09-05.md`,\n`/work/artefacts/archie-core-current-state-archify.html`,\n`/work/artefacts/archie-core-ideal-state-archify.html`\n\nThe Phase 3 UI boundary and configuration decision is ratified in\n[`ui-service-boundary.md`](/dev/prds/ui-service-boundary/). That focused document is\nthe implementation authority for `archie-core-8cda.5.2`; this parent remains\nthe broader service-distribution proposal.\n\n## Decision\n\narchie-core moves from one Go binary (`archied`) hosting every domain\nin-process to a set of independently deployable services, each exposing\nexactly one typed contract. Consumers depend on the contract; they never\nreach into another service's internal types, session state, or config.\ngRPC is the synchronous transport; NATS remains for asynchronous fan-out\n(status, telemetry) rather than being replaced.\n\nThis is a **distribution decision**, not a rewrite of Archie's domain\nmodel. The six-domain split already approved in\n`docs/architecture/index.md` and `docs/architecture/organisation.md`\n(Identity, Agent, Messaging, Workflow, Work Intake, Plugin) is unchanged;\nthis PRD is about how the *processes* running that domain logic get\npackaged, contracted, and discovered -- not about redrawing domain\nboundaries.\n\n### Target services\n\n| Service | Contract it owns | Existing code it replaces/wraps |\n|---|---|---|\n| UI Service | thin client only; talks to Gateway and State Store contracts | `internal/webui` (SPA hosting stays; API surface becomes typed contract clients, not a peer with direct access) |\n| Gateway Service | routing / turn / session contract | `internal/gateway` |\n| Messaging Service | channel-plugin contract (Telegram, email, Discord, Teams, ...) | `internal/channels` |\n| Curator Service | curator contract; **optional**, installs as a Gateway plugin | `internal/domain/curator` |\n| Scheduler Service | cron/ticker contract | `internal/domain/scheduling` |\n| Execution Environment | execution-dispatch contract | `internal/domain/workflow`, task dispatch paths |\n| Runner | isolated worktree/container execution | `internal/container`, `internal/agentexec`, `internal/worktree` |\n| State Store | state-access contract | `internal/store` |\n\n### Discovery\n\nPer `docs/inspiration/service-discovery-research-2026-09-05.md`:\n\n- **Primary:** Kubernetes-native discovery (Service + headless-Service/DNS\n + EndpointSlice, served by CoreDNS), delivered as a Helm chart + Operator,\n on k3s for single-host installs and a real cluster for multi-host or a\n hosted offering.\n- **Fallback:** NATS-based discovery (micro `$SRV` announce + JetStream KV\n registry) for no-K8s single-host installs, with a custom gRPC resolver\n reading `host:port` from the KV bucket.\n- Both implementations sit behind one internal interface:\n\n ```go\n type ServiceRegistry interface {\n Resolve(service string) []Endpoint\n Watch(service string) \u003c-chan Event // join/leave\n }\n ```\n\n- An optional service (Curator; others later) that is not installed\n must resolve to a distinct `NotInstalled` outcome, never a `Down`/\n `Unhealthy` one. Under K8s this falls out of DNS returning no record;\n under the NATS fallback it requires an explicit \"installed\" marker in the\n KV registry, not just absence-implies-down.\n\n### Distribution model this supports\n\nBoth self-hosted (customer runs the Helm chart or the NATS-fallback\nbinaries themselves) and a managed/hosted offering (we run the same Helm\nchart) are in scope -- this is the RHEL / Ansible Automation Platform\nmodel: one architecture, sold both ways. This PRD does not decide the\nbusiness model; it decides that the architecture must not foreclose\neither shape.\n\n## Why this shape\n\n`docs/inspiration/module-reference-awx-decomposition-vs-archie.md`\nestablished that Archie's existing domain split already does what AWX's\nservice-decomposition post argues for -- boundaries exist. What didn't\nexist was deployability: `internal/webui` was not a passive frontend. The\nre-baselined coupling inventory is below, and the ratified boundary is\n[`ui-service-boundary.md`](/dev/prds/ui-service-boundary/).\n\n- Telegram task actions now call Gateway's `ChatContract` directly through\n `chatTaskActorAdapter`; the old Telegram-to-webui HTTP detour is historical.\n- `internal/webui/api_chat.go` now holds `gateway.ChatContract`; direct\n `SessionStore`, `Router`, and `TurnHistory` access is historical.\n- The third coupling is closed. UI extraction shipped (`archie-core-8cda.5`,\n closed 2026-09-11): `cmd/archie-ui` serves the dashboard against remote\n Gateway and State Store contracts, holds no `config.Holder`, and relinks no\n daemon runtime (`cmd/archie-ui/architecture_test.go` enforces that on every\n `task check`). The `b.web.Cfg = b.d.Cfg` alias left in `buildDaemon` is now\n inside the daemon's own process -- it renders the published `ConfigView`\n snapshot and holds the readiness probes -- so it no longer crosses a service\n boundary. That cleanup is `archie-core-ml30` (P2); the UI boundary is\n ratified in [`ui-service-boundary.md`](/dev/prds/ui-service-boundary/).\n\n## Non-goals\n\n- Redesigning the domain model (Identity/Agent/Messaging/Workflow/Work\n Intake/Plugin boundaries are settled elsewhere and unchanged).\n- Replacing NATS. It remains the async transport; this PRD only adds gRPC\n for synchronous request/response and (optionally) K8s-native discovery.\n- Deciding the business model (self-hosted vs. managed vs. both). Kept\n open deliberately; the architecture must serve either.\n- Config relocation (`config.toml` -\u003e ConfigMaps/Secrets) is real work\n implied by a K8s-native deployment but is its own decision, not decided\n here. Flagged in the discovery research as a phased concern.\n- Any concrete adoption of Pi (or another external agent harness) as an\n execution runner. Raised in conversation alongside this decomposition,\n but it is a separate decision (agent execution as a strict data\n boundary, `ARCHITECTURE.md`) and does not gate or get gated by this PRD.\n\n## Open questions\n\nAll ten are resolved; each carries its resolution inline. Three phases have\nsince shipped against them (Gateway, State Store, UI), so the framing here is\nhistorical record rather than an outstanding gate.\n\n1. **Extraction order.** Which service gets pulled out of the monolith\n first? The webui entanglement findings above suggest Gateway Service's\n `ChatContract` should exist before UI Service is extracted, since UI is\n the most entangled consumer. Not yet sequenced.\n\n **RESOLVED.** Gateway Service first, then State Store,\n then UI, then Messaging, then Execution/Runner/Scheduler, then Curator\n (optional) — preceded by an in-process \"contract seams\" (modular\n monolith) phase. Confirms the hypothesis and strengthens it: the\n Gateway is already store-decoupled (it uses narrow contract adapters,\n not `internal/store`) and already owns an isolated session SQLite, so it\n is both the stickiest capability *and* the one with a ready seam and\n isolated data — extractable alone, first. Full phase list and reasoning:\n `docs/inspiration/service-decomposition-open-questions-research-2026-09-05.md#1`.\n2. **Contract definition mechanism.** Protobuf/gRPC service definitions\n need a home (a new `proto/` or `contracts/` tree, versioning scheme,\n codegen wiring into `Taskfile.yml`). Not yet designed.\n\n **RESOLVED.** buf v2 driving `protoc-gen-go` +\n `protoc-gen-go-grpc` (grpc-go, not connect-go — connect-go's net/http\n client breaks the discovery research's gRPC resolver plumbing for K8s\n DNS and NATS-KV). One committed `proto/\u003cservice\u003e/v1/` tree, generated\n Go committed to `internal/contracts/\u003cservice\u003e/v1/`, directory\n versioning (breaking change -\u003e new `vN`, enforced by `buf breaking` in\n the gate), and `task proto:generate`/`proto:lint`/`proto:check` wired\n into `task check` and mirrored into `.github/workflows/deploy.yml`\n (CI does not run `task check`). Full toolchain comparison, layout\n rationale, and scratch-verified `buf`/`buf breaking`/`connect-go`\n results:\n `docs/inspiration/service-decomposition-open-questions-research-2026-09-05.md#2`.\n3. **In-process vs. out-of-process during migration.** Whether services\n can run multiplexed inside one binary behind the same `ServiceRegistry`\n interface during a transition period (so extraction is incremental and\n testable) or whether each service must be a separate binary from day\n one. Strongly recommend the former for a solo maintainer, but not\n decided here.\n\n **RESOLVED.** Multiplexed in-process is the migration\n default -- this is `docs/architecture/organisation.md`'s standing\n \"Process boundaries\" rule, not a new lean. Mechanism: one wire-safe Go\n contract interface per service; a local adapter and a generated gRPC\n client adapter, both satisfying the same interface via a compile-time\n assertion (the `var _ workflow.Store = (*Client)(nil)` idiom;\n `store.WorkflowStore` is superseded by the consumer-owned\n `workflow.Store`); composition (not `ServiceRegistry`, which stays\n network-only) picks the adapter by the presence of a\n `[services.\u003cname\u003e].target` address -- empty = local (in-process) adapter,\n set = dial the remote gRPC client. This replaced the earlier\n `mode = \"inproc\" | \"remote\"` enum, which was dropped (see the gateway's\n `services.gateway.target`). The adapter is chosen by presence of the\n target; Phase 2 then deleted every local fallback, so remote is now the\n only mode and an empty `[services.state].target` is a startup error\n (docs/prds/state-store-contract.md §12 step 7). A\n per-contract conformance suite runs against both adapters over\n `bufconn`; the flip to a real process is a same-commit deletion of the\n in-process path, no dual-live window. Consequence: Gateway's Phase 0\n (`ChatContract` seam) ships in-process first, with the network protocol\n staying out of the production path until Phase 1 -- no amendment to Q1's\n phase list needed. Full mechanism and evidence:\n `docs/inspiration/service-decomposition-open-questions-research-2026-09-05.md#3`\n (the State Store's concrete ratified config shape is\n `docs/prds/state-store-contract.md` §10).\n4. **State Store's contract shape.** Whether every service gets its own\n storage (true microservice isolation) or several services share the\n State Store service's contract (less isolation, less migration risk).\n Not yet decided.\n\n **RESOLVED.** One State Store service owns the existing\n single SQLite file behind narrow typed contracts (`TaskStore`,\n `CaptureStore`, `MappingStore`, `BindingStore`, `WorkflowStore`) over\n gRPC. *(Ratified 2026-09-06: the workflow contract is now the\n domain-owned `workflow.Store`, which supersedes\n `internal/store`'s `WorkflowStore`. See note below and\n `docs/prds/state-store-contract.md`.)*\n No physical database-per-service: the solo-maintainer / team-\n topology rule of thumb plus the low value of per-service autonomy for\n one person rule it out, and one store means no distributed-transactions\n problem. The Gateway keeps its own already-separate session SQLite.\n Concrete migration path (generalise `storerpc`, swap consumers one\n contract at a time, stand up the store service, split the file only on\n evidence):\n `docs/inspiration/service-decomposition-open-questions-research-2026-09-05.md#4`.\n\n **RATIFIED (2026-09-06, beads archie-core-8cda.4.1).** The authoritative\n contract boundary \u0026 transport design for this is fixed in\n `docs/prds/state-store-contract.md` (rev. 2c): one `StateStore` gRPC service,\n 42 RPCs; **ownership split** — consumer-owned `workflow.Store` +\n `workflow.Task`/`Status`/`Source` in the workflow domain (dependency rules\n #2/#7), producer-owned daemon/webui store surfaces in `internal/store`;\n the `Task`/`Status`/`Source` + interface relocation is a Phase 2\n prerequisite pulled from migration-decisions §4 (`store.Task →\n WorkflowExecution`); `RecordDispatch` drops `*sql.Tx`; `Close` off-wire;\n structured gRPC error codes with sanitised details; explicit agent State\n Store gRPC handoff (`STATE_STORE_URL`/`STATE_STORE_TOKEN`); loopback+bearer\n transport-security boundary; presence-based `[services.state].target`\n defaulting local; migration order `workflow.Store -\u003e archie-state-store -\u003e\n Capture -\u003e Mapping -\u003e Binding -\u003e TaskStore`. `.4.2` and `.4.3` implement\n against that doc.\n5. **Helm chart / Operator ownership.** Net-new work with no existing\n analog in this repository (`deployments/` holds TOML profiles\n and a `docker-compose.yml`, not Kubernetes manifests). Scoping not yet\n started.\n\n **RESOLVED.** Three charts: an `archie-library` chart\n (shared Deployment/Service templates, named ports), an `archie` chart\n for the 7 core services, and an `archie-curator` chart + Curator\n Operator (CRD, controller, RBAC). The Curator reconciler is a\n level-triggered loop that fetches the CR, builds Deployment + Service,\n sets owner refs, applies idempotently (`CreateOrUpdate`), watches via\n `Owns()`, and reports a `Ready` status condition. Smallest real\n milestone (M1): install a Curator CR → Operator creates Deployment +\n Service → CoreDNS serves `curator` → Gateway's resolver flips it from\n `NotInstalled` to Installed with no redeploy; uninstall flips it back.\n Full scoping and first bead-issue list:\n `docs/inspiration/service-decomposition-open-questions-research-2026-09-05.md#5`.\n\n## Completion criteria\n\nNot applicable yet -- this PRD records the target shape and the discovery\ndecision. It does not authorize implementation. Before work begins: file\na beads epic, resolve the open questions above (at minimum #1-#3), and\nget explicit go-ahead given the scale of this change relative to\n`AGENTS.md`'s \"smallest workable change\" default for this solo project.\n" + "body": "# Service decomposition -- decision\n\n**Status:** Draft\n**Date:** 2026-09-05\n**Beads epic:** archie-core-8cda\n**Prior art:** `docs/inspiration/2026-09-05-awx-service-decomposition-and-ansible-task-model.md`,\n`docs/inspiration/module-reference-awx-decomposition-vs-archie.md`,\n`docs/inspiration/service-discovery-research-2026-09-05.md`,\n`/work/artefacts/archie-core-current-state-archify.html`,\n`/work/artefacts/archie-core-ideal-state-archify.html`\n\nThe Phase 3 UI boundary and configuration decision is ratified in\n[`ui-service-boundary.md`](/dev/prds/ui-service-boundary/). That focused document is\nthe implementation authority for `archie-core-8cda.5.2`; this parent remains\nthe broader service-distribution proposal.\n\n## Decision\n\narchie-core moves from one Go binary (`archied`) hosting every domain\nin-process to a set of independently deployable services, each exposing\nexactly one typed contract. Consumers depend on the contract; they never\nreach into another service's internal types, session state, or config.\ngRPC is the synchronous transport; NATS remains for asynchronous fan-out\n(status, telemetry) rather than being replaced.\n\nThis is a **distribution decision**, not a rewrite of Archie's domain\nmodel. The six-domain split already approved in\n`docs/architecture/index.md` and `docs/architecture/organisation.md`\n(Identity, Agent, Messaging, Workflow, Work Intake, Plugin) is unchanged;\nthis PRD is about how the *processes* running that domain logic get\npackaged, contracted, and discovered -- not about redrawing domain\nboundaries.\n\n### Target services\n\n| Service | Contract it owns | Existing code it replaces/wraps |\n|---|---|---|\n| UI Service | thin client only; talks to Gateway and State Store contracts | `internal/webui` (SPA hosting stays; API surface becomes typed contract clients, not a peer with direct access) |\n| Gateway Service | routing / turn / session contract | `internal/gateway` |\n| Messaging Service | channel-plugin contract (Telegram, email, Discord, Teams, ...) | `internal/channels` |\n| Curator Service | curator contract; **optional**, installs as a Gateway plugin | `internal/domain/curator` |\n| Scheduler Service | cron/ticker contract | `internal/domain/scheduling` |\n| Execution Environment | execution-dispatch contract | `internal/domain/workflow`, task dispatch paths |\n| Runner | isolated worktree/container execution | `internal/container`, `internal/agentexec`, `internal/worktree` |\n| State Store | state-access contract | `internal/store` |\n\n### Discovery\n\nPer `docs/inspiration/service-discovery-research-2026-09-05.md`:\n\n- **Primary:** Kubernetes-native discovery (Service + headless-Service/DNS\n + EndpointSlice, served by CoreDNS), delivered as a Helm chart + Operator,\n on k3s for single-host installs and a real cluster for multi-host or a\n hosted offering.\n- **Fallback:** NATS-based discovery (micro `$SRV` announce + JetStream KV\n registry) for no-K8s single-host installs, with a custom gRPC resolver\n reading `host:port` from the KV bucket.\n- Both implementations sit behind one internal interface:\n\n ```go\n type ServiceRegistry interface {\n Resolve(service string) []Endpoint\n Watch(service string) \u003c-chan Event // join/leave\n }\n ```\n\n- An optional service (Curator; others later) that is not installed\n must resolve to a distinct `NotInstalled` outcome, never a `Down`/\n `Unhealthy` one. Under K8s this falls out of DNS returning no record;\n under the NATS fallback it requires an explicit \"installed\" marker in the\n KV registry, not just absence-implies-down.\n\n### Distribution model this supports\n\nBoth self-hosted (customer runs the Helm chart or the NATS-fallback\nbinaries themselves) and a managed/hosted offering (we run the same Helm\nchart) are in scope -- this is the RHEL / Ansible Automation Platform\nmodel: one architecture, sold both ways. This PRD does not decide the\nbusiness model; it decides that the architecture must not foreclose\neither shape.\n\n## Why this shape\n\n`docs/inspiration/module-reference-awx-decomposition-vs-archie.md`\nestablished that Archie's existing domain split already does what AWX's\nservice-decomposition post argues for -- boundaries exist. What didn't\nexist was deployability: `internal/webui` was not a passive frontend. The\nre-baselined coupling inventory is below, and the ratified boundary is\n[`ui-service-boundary.md`](/dev/prds/ui-service-boundary/).\n\n- Telegram task actions now call Gateway's `ChatContract` directly through\n `chatTaskActorAdapter`; the old Telegram-to-webui HTTP detour is historical.\n- `internal/webui/api_chat.go` now holds `gateway.ChatContract`; direct\n `SessionStore`, `Router`, and `TurnHistory` access is historical.\n- The third coupling is closed. UI extraction shipped (`archie-core-8cda.5`,\n closed 2026-09-11): `cmd/archie-ui` serves the dashboard against remote\n Gateway and State Store contracts, holds no `config.Holder`, and relinks no\n daemon runtime (`cmd/archie-ui/architecture_test.go` enforces that on every\n `task check`). The `b.web.Cfg = b.d.Cfg` alias left in `buildDaemon` is now\n inside the daemon's own process -- it renders the published `ConfigView`\n snapshot and holds the readiness probes -- so it no longer crosses a service\n boundary. That cleanup is `archie-core-ml30` (P2); the UI boundary is\n ratified in [`ui-service-boundary.md`](/dev/prds/ui-service-boundary/).\n\n## Non-goals\n\n- Redesigning the domain model (Identity/Agent/Messaging/Workflow/Work\n Intake/Plugin boundaries are settled elsewhere and unchanged).\n- Replacing NATS. It remains the async transport; this PRD only adds gRPC\n for synchronous request/response and (optionally) K8s-native discovery.\n- Deciding the business model (self-hosted vs. managed vs. both). Kept\n open deliberately; the architecture must serve either.\n- Config relocation (`config.toml` -\u003e ConfigMaps/Secrets) is real work\n implied by a K8s-native deployment but is its own decision, not decided\n here. Flagged in the discovery research as a phased concern.\n- Any concrete adoption of Pi (or another external agent harness) as an\n execution runner. Raised in conversation alongside this decomposition,\n but it is a separate decision (agent execution as a strict data\n boundary, `docs/architecture/agent-system.md`) and does not gate or get gated by this PRD.\n\n## Open questions\n\nAll ten are resolved; each carries its resolution inline. Three phases have\nsince shipped against them (Gateway, State Store, UI), so the framing here is\nhistorical record rather than an outstanding gate.\n\n1. **Extraction order.** Which service gets pulled out of the monolith\n first? The webui entanglement findings above suggest Gateway Service's\n `ChatContract` should exist before UI Service is extracted, since UI is\n the most entangled consumer. Not yet sequenced.\n\n **RESOLVED.** Gateway Service first, then State Store,\n then UI, then Messaging, then Execution/Runner/Scheduler, then Curator\n (optional) — preceded by an in-process \"contract seams\" (modular\n monolith) phase. Confirms the hypothesis and strengthens it: the\n Gateway is already store-decoupled (it uses narrow contract adapters,\n not `internal/store`) and already owns an isolated session SQLite, so it\n is both the stickiest capability *and* the one with a ready seam and\n isolated data — extractable alone, first. Full phase list and reasoning:\n `docs/inspiration/service-decomposition-open-questions-research-2026-09-05.md#1`.\n2. **Contract definition mechanism.** Protobuf/gRPC service definitions\n need a home (a new `proto/` or `contracts/` tree, versioning scheme,\n codegen wiring into `Taskfile.yml`). Not yet designed.\n\n **RESOLVED.** buf v2 driving `protoc-gen-go` +\n `protoc-gen-go-grpc` (grpc-go, not connect-go — connect-go's net/http\n client breaks the discovery research's gRPC resolver plumbing for K8s\n DNS and NATS-KV). One committed `proto/\u003cservice\u003e/v1/` tree, generated\n Go committed to `internal/contracts/\u003cservice\u003e/v1/`, directory\n versioning (breaking change -\u003e new `vN`, enforced by `buf breaking` in\n the gate), and `task proto:generate`/`proto:lint`/`proto:check` wired\n into `task check` and mirrored into `.github/workflows/deploy.yml`\n (CI does not run `task check`). Full toolchain comparison, layout\n rationale, and scratch-verified `buf`/`buf breaking`/`connect-go`\n results:\n `docs/inspiration/service-decomposition-open-questions-research-2026-09-05.md#2`.\n3. **In-process vs. out-of-process during migration.** Whether services\n can run multiplexed inside one binary behind the same `ServiceRegistry`\n interface during a transition period (so extraction is incremental and\n testable) or whether each service must be a separate binary from day\n one. Strongly recommend the former for a solo maintainer, but not\n decided here.\n\n **RESOLVED.** Multiplexed in-process is the migration\n default -- this is `docs/architecture/organisation.md`'s standing\n \"Process boundaries\" rule, not a new lean. Mechanism: one wire-safe Go\n contract interface per service; a local adapter and a generated gRPC\n client adapter, both satisfying the same interface via a compile-time\n assertion (the `var _ workflow.Store = (*Client)(nil)` idiom;\n `store.WorkflowStore` is superseded by the consumer-owned\n `workflow.Store`); composition (not `ServiceRegistry`, which stays\n network-only) picks the adapter by the presence of a\n `[services.\u003cname\u003e].target` address -- empty = local (in-process) adapter,\n set = dial the remote gRPC client. This replaced the earlier\n `mode = \"inproc\" | \"remote\"` enum, which was dropped (see the gateway's\n `services.gateway.target`). The adapter is chosen by presence of the\n target; Phase 2 then deleted every local fallback, so remote is now the\n only mode and an empty `[services.state].target` is a startup error\n (docs/prds/state-store-contract.md §12 step 7). A\n per-contract conformance suite runs against both adapters over\n `bufconn`; the flip to a real process is a same-commit deletion of the\n in-process path, no dual-live window. Consequence: Gateway's Phase 0\n (`ChatContract` seam) ships in-process first, with the network protocol\n staying out of the production path until Phase 1 -- no amendment to Q1's\n phase list needed. Full mechanism and evidence:\n `docs/inspiration/service-decomposition-open-questions-research-2026-09-05.md#3`\n (the State Store's concrete ratified config shape is\n `docs/prds/state-store-contract.md` §10).\n4. **State Store's contract shape.** Whether every service gets its own\n storage (true microservice isolation) or several services share the\n State Store service's contract (less isolation, less migration risk).\n Not yet decided.\n\n **RESOLVED.** One State Store service owns the existing\n single SQLite file behind narrow typed contracts (`TaskStore`,\n `CaptureStore`, `MappingStore`, `BindingStore`, `WorkflowStore`) over\n gRPC. *(Ratified 2026-09-06: the workflow contract is now the\n domain-owned `workflow.Store`, which supersedes\n `internal/store`'s `WorkflowStore`. See note below and\n `docs/prds/state-store-contract.md`.)*\n No physical database-per-service: the solo-maintainer / team-\n topology rule of thumb plus the low value of per-service autonomy for\n one person rule it out, and one store means no distributed-transactions\n problem. The Gateway keeps its own already-separate session SQLite.\n Concrete migration path (generalise `storerpc`, swap consumers one\n contract at a time, stand up the store service, split the file only on\n evidence):\n `docs/inspiration/service-decomposition-open-questions-research-2026-09-05.md#4`.\n\n **RATIFIED (2026-09-06, beads archie-core-8cda.4.1).** The authoritative\n contract boundary \u0026 transport design for this is fixed in\n `docs/prds/state-store-contract.md` (rev. 2c): one `StateStore` gRPC service,\n 42 RPCs; **ownership split** — consumer-owned `workflow.Store` +\n `workflow.Task`/`Status`/`Source` in the workflow domain (dependency rules\n #2/#7), producer-owned daemon/webui store surfaces in `internal/store`;\n the `Task`/`Status`/`Source` + interface relocation is a Phase 2\n prerequisite pulled from migration-decisions §4 (`store.Task →\n WorkflowExecution`); `RecordDispatch` drops `*sql.Tx`; `Close` off-wire;\n structured gRPC error codes with sanitised details; explicit agent State\n Store gRPC handoff (`STATE_STORE_URL`/`STATE_STORE_TOKEN`); loopback+bearer\n transport-security boundary; presence-based `[services.state].target`\n defaulting local; migration order `workflow.Store -\u003e archie-state-store -\u003e\n Capture -\u003e Mapping -\u003e Binding -\u003e TaskStore`. `.4.2` and `.4.3` implement\n against that doc.\n5. **Helm chart / Operator ownership.** Net-new work with no existing\n analog in this repository (`deployments/` holds TOML profiles\n and a `docker-compose.yml`, not Kubernetes manifests). Scoping not yet\n started.\n\n **RESOLVED.** Three charts: an `archie-library` chart\n (shared Deployment/Service templates, named ports), an `archie` chart\n for the 7 core services, and an `archie-curator` chart + Curator\n Operator (CRD, controller, RBAC). The Curator reconciler is a\n level-triggered loop that fetches the CR, builds Deployment + Service,\n sets owner refs, applies idempotently (`CreateOrUpdate`), watches via\n `Owns()`, and reports a `Ready` status condition. Smallest real\n milestone (M1): install a Curator CR → Operator creates Deployment +\n Service → CoreDNS serves `curator` → Gateway's resolver flips it from\n `NotInstalled` to Installed with no redeploy; uninstall flips it back.\n Full scoping and first bead-issue list:\n `docs/inspiration/service-decomposition-open-questions-research-2026-09-05.md#5`.\n\n## Completion criteria\n\nNot applicable yet -- this PRD records the target shape and the discovery\ndecision. It does not authorize implementation. Before work begins: file\na beads epic, resolve the open questions above (at minimum #1-#3), and\nget explicit go-ahead given the scale of this change relative to\n`AGENTS.md`'s \"smallest workable change\" default for this solo project.\n" }, { "url": "/dev/prds/service-registry/", @@ -424,7 +424,7 @@ "url": "/dev/prds/webhook-intake-security/", "title": "Webhook intake security -- decision", "section": "PRDs", - "body": "# Webhook intake security -- decision\n\n**Status:** Approved\n**Date:** 2026-08-22\n**Beads issue:** `archie-core-t2db.5`, blocks `archie-core-t2db.4`\n\n## Why this is decided ahead of the code it constrains\n\n`archie-core-t2db` (no-code playbooks) and `archie-core-7d5u` (typed event\nsources) both land a public intake surface whose eventual output is an agent\nwith tool access and a repo. Neither capture (`t2db.1`) nor the intake family\n(`7d5u.3`) has shipped yet, so this decision exists to be a contract the two\nepics build against, not a retrofit once the surface exists. A webhook URL\nthat can start an agent is remote code execution with extra steps if any of\nthe five points below is missing on day one.\n\n## 1. Per-source authentication\n\nReuse the per-route HMAC-SHA256 scheme already in\n`internal/channels/webhook/webhook.go` (`validateHMAC`), unchanged: a source\nregisters with a shared secret; the signature is checked against the raw\nbody. Do not invent a second scheme for intake.\n\n**Capture and triggering split at the signature, not at a separate flag.** An\nevent with a missing or invalid signature is still captured (visible in the\ninspector, marked unauthenticated) so an operator can diagnose a\nmisconfigured sender -- but an unauthenticated event can never be matched\nagainst a binding. Authentication is what promotes an event from \"something\narrived\" to \"something that can run a workflow.\"\n\n## 2. Human approval before a binding arms\n\nA binding has three states: `draft -\u003e pending_approval -\u003e armed`. Only\n`armed` bindings are evaluated against incoming events. Creating or editing a\nbinding's matcher, mapping, or target workflow always sets it to\n`pending_approval`, including edits to an already-armed binding -- an edit is\nnot silently re-armed, or the approval gate is decorative. Approval requires\nan authenticated dashboard session; there is no unattended approval path.\n\nModel: `internal/channels/telegram`'s dangerous-command approval flow\n(`dangerousAction` / `pendingApproval` in `dangerous.go` / `approval.go`) is\nthe existing precedent for \"a human decision gates code from running\" in this\ncodebase -- follow its shape rather than designing a new approval primitive.\n\n## 3. Rate and volume limits, per source\n\nA token-bucket limiter keyed by source ID, not global -- one noisy or hostile\nsender must not exhaust another source's capture budget. Precedent for\nper-key volume capping: `internal/pairing/store.go`'s `MaxPendingPerPlatform`.\n\nOn limit exceeded, the request is rejected (HTTP 429), not silently dropped.\nA silently dropped event is indistinguishable from a sender that never fired\n-- the same invisible-failure shape the update-verification work\n(`archie-core-522`) exists to prevent. The rejection itself is observable in\nthe inspector as a distinct outcome from \"not received.\"\n\n## 4. Blast radius bounded by code, not by prompt wording\n\nA playbook-triggered task runs through exactly the same enforcement as every\nother task: the `[[repos.gate]]` quality gate, `diff_cap_lines`, the\nsandboxed per-task container (`internal/container`), worktree isolation, and\n\"the model never runs git.\" No new relaxation is introduced for\nplaybook-triggered work, and none is needed -- the existing constraints are\nthe control.\n\nMapped payload fields are treated as ordinary user-controlled text inside an\nagent prompt, exactly like an inbound chat message never as trusted\ninstructions. \"The agent should ignore instructions in the payload\" is not a\ncontrol and must not appear as one anywhere in this feature -- per\n`ARCHITECTURE.md`'s environmental-enforcement-over-prompt-rules principle,\nthe boundary is the gate and the sandbox, not a sentence in the system\nprompt.\n\n## 5. Retention and redaction of captured payloads\n\n**Retention:** captured events (bound or not) are purged after 7 days,\nconfigurable per deployment. Unbound events exist to be mapped against; nothing\njustifies keeping them indefinitely once that window has passed, and an\nunbounded capture store is the same disk-exhaustion risk item 3 addresses at\nthe request-rate level.\n\n**Redaction:** before a captured payload is persisted, apply the same\nkey-name heuristic `internal/gateway/stream.go`'s `sensitiveParameterKey`\nalready uses for tool-call parameters (matches `token`, `secret`, `password`,\n`passwd`, `api_key`, `apikey`, `authorization`, `credential`, `cookie`,\n`private_key` against normalized JSON keys) and replace matching values with\nthe same `[redacted]` marker. This is best-effort, not a guarantee -- a\nsender that names a secret field something the heuristic misses will still\nhave it captured. Document that limitation where captured payloads are\ndisplayed, rather than implying the redaction is complete.\n\n**Visibility:** captured payloads, redacted or not, are visible only to an\nauthenticated dashboard operator -- the same authorization boundary as every\nother webui surface, not a separate or lower bar.\n\n## What this does not decide\n\nThe intake family's registration interface (`7d5u.3`), the capture storage\nmechanism (`7d5u.2`, embedded NATS), and the binding data model's persistence\nand versioning (`t2db.4`) are separate decisions. This document constrains\nall three: whatever they build must satisfy points 1-5 above.\n" + "body": "# Webhook intake security -- decision\n\n**Status:** Approved\n**Date:** 2026-08-22\n**Beads issue:** `archie-core-t2db.5`, blocks `archie-core-t2db.4`\n\n## Why this is decided ahead of the code it constrains\n\n`archie-core-t2db` (no-code playbooks) and `archie-core-7d5u` (typed event\nsources) both land a public intake surface whose eventual output is an agent\nwith tool access and a repo. Neither capture (`t2db.1`) nor the intake family\n(`7d5u.3`) has shipped yet, so this decision exists to be a contract the two\nepics build against, not a retrofit once the surface exists. A webhook URL\nthat can start an agent is remote code execution with extra steps if any of\nthe five points below is missing on day one.\n\n## 1. Per-source authentication\n\nReuse the per-route HMAC-SHA256 scheme already in\n`internal/channels/webhook/webhook.go` (`validateHMAC`), unchanged: a source\nregisters with a shared secret; the signature is checked against the raw\nbody. Do not invent a second scheme for intake.\n\n**Capture and triggering split at the signature, not at a separate flag.** An\nevent with a missing or invalid signature is still captured (visible in the\ninspector, marked unauthenticated) so an operator can diagnose a\nmisconfigured sender -- but an unauthenticated event can never be matched\nagainst a binding. Authentication is what promotes an event from \"something\narrived\" to \"something that can run a workflow.\"\n\n## 2. Human approval before a binding arms\n\nA binding has three states: `draft -\u003e pending_approval -\u003e armed`. Only\n`armed` bindings are evaluated against incoming events. Creating or editing a\nbinding's matcher, mapping, or target workflow always sets it to\n`pending_approval`, including edits to an already-armed binding -- an edit is\nnot silently re-armed, or the approval gate is decorative. Approval requires\nan authenticated dashboard session; there is no unattended approval path.\n\nModel: `internal/channels/telegram`'s dangerous-command approval flow\n(`dangerousAction` / `pendingApproval` in `dangerous.go` / `approval.go`) is\nthe existing precedent for \"a human decision gates code from running\" in this\ncodebase -- follow its shape rather than designing a new approval primitive.\n\n## 3. Rate and volume limits, per source\n\nA token-bucket limiter keyed by source ID, not global -- one noisy or hostile\nsender must not exhaust another source's capture budget. Precedent for\nper-key volume capping: `internal/pairing/store.go`'s `MaxPendingPerPlatform`.\n\nOn limit exceeded, the request is rejected (HTTP 429), not silently dropped.\nA silently dropped event is indistinguishable from a sender that never fired\n-- the same invisible-failure shape the update-verification work\n(`archie-core-522`) exists to prevent. The rejection itself is observable in\nthe inspector as a distinct outcome from \"not received.\"\n\n## 4. Blast radius bounded by code, not by prompt wording\n\nA playbook-triggered task runs through exactly the same enforcement as every\nother task: the `[[repos.gate]]` quality gate, `diff_cap_lines`, the\nsandboxed per-task container (`internal/container`), worktree isolation, and\n\"the model never runs git.\" No new relaxation is introduced for\nplaybook-triggered work, and none is needed -- the existing constraints are\nthe control.\n\nMapped payload fields are treated as ordinary user-controlled text inside an\nagent prompt, exactly like an inbound chat message never as trusted\ninstructions. \"The agent should ignore instructions in the payload\" is not a\ncontrol and must not appear as one anywhere in this feature -- per the\nenvironmental-enforcement principle (`docs/architecture/agent-system.md`,\n\"Existing invariants to preserve\"), the boundary is the gate and the sandbox,\nnot a sentence in the system prompt.\n\n## 5. Retention and redaction of captured payloads\n\n**Retention:** captured events (bound or not) are purged after 7 days,\nconfigurable per deployment. Unbound events exist to be mapped against; nothing\njustifies keeping them indefinitely once that window has passed, and an\nunbounded capture store is the same disk-exhaustion risk item 3 addresses at\nthe request-rate level.\n\n**Redaction:** before a captured payload is persisted, apply the same\nkey-name heuristic `internal/gateway/stream.go`'s `sensitiveParameterKey`\nalready uses for tool-call parameters (matches `token`, `secret`, `password`,\n`passwd`, `api_key`, `apikey`, `authorization`, `credential`, `cookie`,\n`private_key` against normalized JSON keys) and replace matching values with\nthe same `[redacted]` marker. This is best-effort, not a guarantee -- a\nsender that names a secret field something the heuristic misses will still\nhave it captured. Document that limitation where captured payloads are\ndisplayed, rather than implying the redaction is complete.\n\n**Visibility:** captured payloads, redacted or not, are visible only to an\nauthenticated dashboard operator -- the same authorization boundary as every\nother webui surface, not a separate or lower bar.\n\n## What this does not decide\n\nThe intake family's registration interface (`7d5u.3`), the capture storage\nmechanism (`7d5u.2`, embedded NATS), and the binding data model's persistence\nand versioning (`t2db.4`) are separate decisions. This document constrains\nall three: whatever they build must satisfy points 1-5 above.\n" } ], "nav": [ diff --git a/docs/data/generated/docs.json b/docs/data/generated/docs.json index b56e5b667..809f54f47 100644 --- a/docs/data/generated/docs.json +++ b/docs/data/generated/docs.json @@ -10,7 +10,7 @@ "url": "/docs/guides/first-playbook/", "title": "Building your first playbook binding", "section": "Guides", - "body": "# Building your first playbook binding\n\nThis walks through the whole no-code path the event-capture epic promises:\npoint an external app's webhook at archie, see the real payload it sends,\nmap the fields you care about, bind them to a workflow, and watch the next\nmatching event turn into a task -- no code, no redeploy, no restart. See\n`docs/architecture/bindings.md` for the model and threat model behind each\nstep below.\n\n## 1. Send a test event\n\nPoint whatever external app you want archie to react to at:\n\n```\nPOST http://\u003cyour-archied-host\u003e:8484/webhooks/capture/\u003csource\u003e\n```\n\n`\u003csource\u003e` is a name you choose for this sender (e.g. `sentry`, `github-ci`,\n`stripe`) -- it's just a URL path segment, not a preset list. Send one real\nevent however is easiest for a first test, e.g.:\n\n```bash\ncurl -X POST http://localhost:8484/webhooks/capture/sentry \\\n -H \"Content-Type: application/json\" \\\n -d '{\"issue\": {\"title\": \"NullPointerException in checkout\"}, \"level\": \"error\"}'\n```\n\nThis event isn't signed yet, so it's captured but marked **unauthenticated**\n-- visible for you to design against, but incapable of triggering anything\n(`docs/architecture/bindings.md`'s threat-model section explains why: an\nunauthenticated event can never match a binding, no matter how it looks).\n\n## 2. Inspect the captured payload\n\nOpen the dashboard's **Event inspector** (`/captures`). Your test event\nappears there with its real body -- this is schema-by-example: you're\nlooking at an actual payload from the actual sender, not guessing at a\nschema in advance.\n\n## 3. Map the fields you care about\n\nOpen **Field mappings** (`/mappings`) → **New mapping**. Pick your captured\nevent from the dropdown, then click through its payload tree on the right --\nevery key and value is clickable. Clicking a value binds it as a field\n(name, JSON path, and an inferred type you can change). Add whichever fields\nyour workflow needs (e.g. the issue title), mark the ones that must be\npresent as **Required**, hit **Preview** to confirm they resolve correctly\nagainst the real payload, then **Save**.\n\n## 4. Create the binding\n\nOpen **Playbook bindings** (`/bindings`) → **New binding**:\n\n- **Name** -- anything descriptive.\n- **Source** -- the same `\u003csource\u003e` segment from step 1 (e.g. `sentry`).\n- **Field mapping** -- the one you just saved.\n- **Workflow** -- which registered workflow should run on a match.\n- **Repo pin** (optional) -- leave both blank to use the single configured\n repo; set owner/repo explicitly if you run a multi-repo deployment.\n- **Shared secret** -- a real secret (16+ bytes), never `test1234`. The\n sender signs every request body with this via HMAC-SHA256 so archie can\n tell a genuine event from anyone who finds the URL.\n\nSave. The binding starts in **draft** -- nothing fires yet.\n\n## 5. Approve it\n\nA **draft** or freshly-**edited** binding needs an explicit human approval\nbefore it can ever match a live event -- this is deliberate, not a missing\nstep: an operator authoring a binding is a considered act, an event arriving\nis not, and nothing should self-arm (`docs/architecture/bindings.md`'s\nthreat-model section, point 2). Click **Approve**. Status becomes **armed**.\n\n## 6. Fire it for real\n\nSend the same kind of event again, this time **signed** with the binding's\nsecret:\n\n```bash\nBODY='{\"issue\": {\"title\": \"NullPointerException in checkout\"}, \"level\": \"error\"}'\nSECRET=\"the-secret-you-set-in-step-4\"\nSIG=$(printf '%s' \"$BODY\" | openssl dgst -sha256 -hmac \"$SECRET\" | sed 's/^.* //')\n\ncurl -X POST http://localhost:8484/webhooks/capture/sentry \\\n -H \"Content-Type: application/json\" \\\n -H \"X-Hub-Signature-256: sha256=$SIG\" \\\n -d \"$BODY\"\n```\n\nThis event is captured as **authenticated**, matched against your armed\nbinding, and dispatched into a new task through the exact same\ngate/worktree/sandbox pipeline every other task goes through -- see\n`ARCHITECTURE.md`'s Task Lifecycle section, \"Provenance by origin.\" Check\nthe **Tasks** page; a new task should appear, its body built from the fields\nyour mapping resolved.\n\n## Editing a live binding\n\nAny edit to an armed binding -- even a typo fix -- drops it back to\n**pending_approval** automatically. This can't be bypassed by editing; it\nhas to be re-approved. When editing, leave the **Shared secret** field blank\nto keep the existing secret; only fill it in if you're deliberately\nrotating it.\n" + "body": "# Building your first playbook binding\n\nThis walks through the whole no-code path the event-capture epic promises:\npoint an external app's webhook at archie, see the real payload it sends,\nmap the fields you care about, bind them to a workflow, and watch the next\nmatching event turn into a task -- no code, no redeploy, no restart. See\n`docs/architecture/bindings.md` for the model and threat model behind each\nstep below.\n\n## 1. Send a test event\n\nPoint whatever external app you want archie to react to at:\n\n```\nPOST http://\u003cyour-archied-host\u003e:8484/webhooks/capture/\u003csource\u003e\n```\n\n`\u003csource\u003e` is a name you choose for this sender (e.g. `sentry`, `github-ci`,\n`stripe`) -- it's just a URL path segment, not a preset list. Send one real\nevent however is easiest for a first test, e.g.:\n\n```bash\ncurl -X POST http://localhost:8484/webhooks/capture/sentry \\\n -H \"Content-Type: application/json\" \\\n -d '{\"issue\": {\"title\": \"NullPointerException in checkout\"}, \"level\": \"error\"}'\n```\n\nThis event isn't signed yet, so it's captured but marked **unauthenticated**\n-- visible for you to design against, but incapable of triggering anything\n(`docs/architecture/bindings.md`'s threat-model section explains why: an\nunauthenticated event can never match a binding, no matter how it looks).\n\n## 2. Inspect the captured payload\n\nOpen the dashboard's **Event inspector** (`/captures`). Your test event\nappears there with its real body -- this is schema-by-example: you're\nlooking at an actual payload from the actual sender, not guessing at a\nschema in advance.\n\n## 3. Map the fields you care about\n\nOpen **Field mappings** (`/mappings`) → **New mapping**. Pick your captured\nevent from the dropdown, then click through its payload tree on the right --\nevery key and value is clickable. Clicking a value binds it as a field\n(name, JSON path, and an inferred type you can change). Add whichever fields\nyour workflow needs (e.g. the issue title), mark the ones that must be\npresent as **Required**, hit **Preview** to confirm they resolve correctly\nagainst the real payload, then **Save**.\n\n## 4. Create the binding\n\nOpen **Playbook bindings** (`/bindings`) → **New binding**:\n\n- **Name** -- anything descriptive.\n- **Source** -- the same `\u003csource\u003e` segment from step 1 (e.g. `sentry`).\n- **Field mapping** -- the one you just saved.\n- **Workflow** -- which registered workflow should run on a match.\n- **Repo pin** (optional) -- leave both blank to use the single configured\n repo; set owner/repo explicitly if you run a multi-repo deployment.\n- **Shared secret** -- a real secret (16+ bytes), never `test1234`. The\n sender signs every request body with this via HMAC-SHA256 so archie can\n tell a genuine event from anyone who finds the URL.\n\nSave. The binding starts in **draft** -- nothing fires yet.\n\n## 5. Approve it\n\nA **draft** or freshly-**edited** binding needs an explicit human approval\nbefore it can ever match a live event -- this is deliberate, not a missing\nstep: an operator authoring a binding is a considered act, an event arriving\nis not, and nothing should self-arm (`docs/architecture/bindings.md`'s\nthreat-model section, point 2). Click **Approve**. Status becomes **armed**.\n\n## 6. Fire it for real\n\nSend the same kind of event again, this time **signed** with the binding's\nsecret:\n\n```bash\nBODY='{\"issue\": {\"title\": \"NullPointerException in checkout\"}, \"level\": \"error\"}'\nSECRET=\"the-secret-you-set-in-step-4\"\nSIG=$(printf '%s' \"$BODY\" | openssl dgst -sha256 -hmac \"$SECRET\" | sed 's/^.* //')\n\ncurl -X POST http://localhost:8484/webhooks/capture/sentry \\\n -H \"Content-Type: application/json\" \\\n -H \"X-Hub-Signature-256: sha256=$SIG\" \\\n -d \"$BODY\"\n```\n\nThis event is captured as **authenticated**, matched against your armed\nbinding, and dispatched into a new task through the exact same\ngate/worktree/sandbox pipeline every other task goes through -- see\n`docs/architecture/messaging-and-work-intake.md`'s \"Intake provenance\". Check\nthe **Tasks** page; a new task should appear, its body built from the fields\nyour mapping resolved.\n\n## Editing a live binding\n\nAny edit to an armed binding -- even a typo fix -- drops it back to\n**pending_approval** automatically. This can't be bypassed by editing; it\nhas to be re-approved. When editing, leave the **Shared secret** field blank\nto keep the existing secret; only fill it in if you're deliberately\nrotating it.\n" } ], "nav": [ diff --git a/docs/guides/first-playbook.md b/docs/guides/first-playbook.md index 27e259006..022209b2c 100644 --- a/docs/guides/first-playbook.md +++ b/docs/guides/first-playbook.md @@ -90,7 +90,7 @@ curl -X POST http://localhost:8484/webhooks/capture/sentry \ This event is captured as **authenticated**, matched against your armed binding, and dispatched into a new task through the exact same gate/worktree/sandbox pipeline every other task goes through -- see -`ARCHITECTURE.md`'s Task Lifecycle section, "Provenance by origin." Check +`docs/architecture/messaging-and-work-intake.md`'s "Intake provenance". Check the **Tasks** page; a new task should appear, its body built from the fields your mapping resolved. diff --git a/docs/prds/deep-research-capability.md b/docs/prds/deep-research-capability.md index ac7a383a8..fecb27e57 100644 --- a/docs/prds/deep-research-capability.md +++ b/docs/prds/deep-research-capability.md @@ -25,7 +25,7 @@ Every claim in this document is mapped to what exists in the tree. | Tool guardrails (repeat-failure / no-progress) | **Implemented, not on the call path** | `internal/tools/guardrail.go` engine exists. Its only non-test caller is `dispatchOne` — which is itself unreachable — so **the chat path never consults it**, and the agent path only records per *stage* (`domain/workflow/agent.go`) | | Structured capture tools (model reports into a typed store, not prose) | **Implemented** | `internal/app/agentworker/review.go` `record_finding` / `record_checked` — `core.NewTypedTool` appending to a caller-owned slice. This is the pattern a research source ledger should reuse | | Curator engine (interval, declared tool set, lifecycle) | **Implemented, tools broken** | `internal/domain/curator`; a curator declaring tools is **refused** at registration (`registry.go`, no `ToolBuilder` implementation) and `curatorLLMRunner` (`internal/app/archied/main.go`) drops `ChatRequest.Tools`. Budgets are time+concurrency only (`runtime.go`) — **no token budget** | -| Cron / scheduling + delivery router | **Implemented, wired into no binary** | `internal/domain/scheduling`, `internal/infrastructure/cronstore` (schema v2, `Kind` discriminator), `internal/infrastructure/crondelivery` — all present and green, but `go list -deps` for every `cmd/` contains none of them, and `cron`-expression firing arithmetic is unimplemented (`schedule.go`, "slice 2"). `ARCHITECTURE.md` still asserts "**No daemon-internal cron.**" | +| Cron / scheduling + delivery router | **Implemented, wired into the daemon** | `internal/domain/scheduling` (ticker engine), `internal/infrastructure/cronstore` (schema v2, `Kind` discriminator), and `internal/infrastructure/crondelivery`; `internal/app/archied/scheduling.go` composes them and `main.go` starts the engine. | | Workflow engines (multi-stage, resumable) | **Implemented, worktree-bound** | `internal/domain/workflow` — every non-triage workflow opens with `StagePrepareWorktree()` and mounts `tc.Dir` into a container; the task persists after every stage (`workflow.go`). `triage.go` documents there is "no cheaper directory-less classification primitive" | | Long-running chat-initiated work with progress | **Partial** | Chat turns run **once, synchronously**, inline in the channel's turn runner (`internal/gateway/turn.go`). Chat can spawn a *workflow task* (`task_spawn`) which is asynchronous — but that path is worktree/PR-shaped | | **Operator-push delivery of an async result** | **Aspirational — zero code** | Nothing pushes a finished result to a chat. No channel subscribes to the event bus (Telegram only *publishes* `KindTurnCompleted`); `crondelivery.Courier` is the intended seam but has **no implementation and no caller**; `send_file` works only inside the live turn. The one out-of-band notification in the tree is feasibility's n8n webhook (`feasibility.go`) | @@ -198,7 +198,7 @@ Reusing it would mean every ad-hoc research run leaves a permanent job row. | Collator | `internal/domain/research` | Model pass over `[]ChildConclusion` + source store → report body | | Report renderer | `internal/domain/research` | Markdown + numbered citation list rendered **from the store** | | Chat trigger | slice 2: `internal/tools/research`; slice 3: gateway tool + courier in `internal/app/archied` | — | -| Durable run + delivery (slice 3) | `internal/domain/research` + `internal/infrastructure/research` + `internal/app/archied` wiring | Follows the strict plugin-engine rule in `ARCHITECTURE.md` ("Plugin engine +| Durable run + delivery (slice 3) | `internal/domain/research` + `internal/infrastructure/research` + `internal/app/archied` wiring | Follows the strict plugin-engine rule in `docs/architecture/plugins-and-extensions.md` ("Plugin engine rule (strict)"): typed contract, owning registry with start/health/stop, narrow typed registrar | ### Why not the curator diff --git a/docs/prds/eda-playbook-engine.md b/docs/prds/eda-playbook-engine.md index f7a55124f..7e515c8c8 100644 --- a/docs/prds/eda-playbook-engine.md +++ b/docs/prds/eda-playbook-engine.md @@ -42,7 +42,8 @@ long as it's implemented behind one of the typed engine families below. generic interface -- see "Typed positions, not one generic hook" below, which is how it stays compliant with the plugin engine rule instead of reopening that document's #4. -- `ARCHITECTURE.md#plugin-engine-rule-strict` governs every extension point +- `docs/architecture/plugins-and-extensions.md#plugin-engine-rule-strict` + governs every extension point introduced here. Every position below is a typed domain contract with an owning registry, not a generic callback map. This is a hard constraint, confirmed against the existing rule text before writing this doc. diff --git a/docs/prds/event-sources-and-reactions.md b/docs/prds/event-sources-and-reactions.md index 71ba40d0f..0f9b93f71 100644 --- a/docs/prds/event-sources-and-reactions.md +++ b/docs/prds/event-sources-and-reactions.md @@ -12,7 +12,8 @@ each question and saying which packages change. ## 1. In-process typed families, not out-of-process dispatch Archie has no out-of-process extension mechanism. The plugin engine -rule (`ARCHITECTURE.md#plugin-engine-rule-strict`, mechanically checked by +rule (`docs/architecture/plugins-and-extensions.md#plugin-engine-rule-strict`, +mechanically checked by `internal/plugin/architecture_test.go`) keeps every capability family in-process behind a narrow typed registrar; NATS RPC exists for daemon<->agentexec, a different actor with a different trust boundary (task diff --git a/docs/prds/image-capability-contract.md b/docs/prds/image-capability-contract.md index c9399e445..1950b2327 100644 --- a/docs/prds/image-capability-contract.md +++ b/docs/prds/image-capability-contract.md @@ -25,8 +25,9 @@ in the repo defines this (`grep -ri image` over `internal/domain` and ## Design Follows the plugin engine family shape already established by -`internal/domain/curator` and `internal/domain/memory` (ARCHITECTURE.md -"plugin engine rule strict"): a typed contract with real domain operations, +`internal/domain/curator` and `internal/domain/memory` +(`docs/architecture/plugins-and-extensions.md#plugin-engine-rule-strict`): a +typed contract with real domain operations, an owning `Registry` (register/get/names, no start/stop lifecycle — see below), and narrow host access via a `Registrar`. diff --git a/docs/prds/memory-engine-unification.md b/docs/prds/memory-engine-unification.md index 89ecaee07..e5d5deb66 100644 --- a/docs/prds/memory-engine-unification.md +++ b/docs/prds/memory-engine-unification.md @@ -332,7 +332,8 @@ Each step is independently verifiable and lands green on its own. 4. **Chat write path.** The tool adapter onto CRUD and the turn's subject on the context. *Depends on 3, because a write is only observable through the read.* 5. **Delete `internal/memory` and the dead config/dashboard surface**, and - rewrite `ARCHITECTURE.md`'s "Memory engine family" section to describe the + rewrite the "Memory engine family" section of + `docs/architecture/plugins-and-extensions.md` to describe the engine family rather than a manager it deleted. *Depends on 3 and 4.* Steps 3 and 4 are separable from each other; step 1 is separable from everything. diff --git a/docs/prds/service-decomposition.md b/docs/prds/service-decomposition.md index 5468e16c0..eee35bad0 100644 --- a/docs/prds/service-decomposition.md +++ b/docs/prds/service-decomposition.md @@ -116,7 +116,7 @@ re-baselined coupling inventory is below, and the ratified boundary is - Any concrete adoption of Pi (or another external agent harness) as an execution runner. Raised in conversation alongside this decomposition, but it is a separate decision (agent execution as a strict data - boundary, `ARCHITECTURE.md`) and does not gate or get gated by this PRD. + boundary, `docs/architecture/agent-system.md`) and does not gate or get gated by this PRD. ## Open questions diff --git a/docs/prds/webhook-intake-security.md b/docs/prds/webhook-intake-security.md index c2e36ded3..0f8604e15 100644 --- a/docs/prds/webhook-intake-security.md +++ b/docs/prds/webhook-intake-security.md @@ -66,10 +66,10 @@ the control. Mapped payload fields are treated as ordinary user-controlled text inside an agent prompt, exactly like an inbound chat message never as trusted instructions. "The agent should ignore instructions in the payload" is not a -control and must not appear as one anywhere in this feature -- per -`ARCHITECTURE.md`'s environmental-enforcement-over-prompt-rules principle, -the boundary is the gate and the sandbox, not a sentence in the system -prompt. +control and must not appear as one anywhere in this feature -- per the +environmental-enforcement principle (`docs/architecture/agent-system.md`, +"Existing invariants to preserve"), the boundary is the gate and the sandbox, +not a sentence in the system prompt. ## 5. Retention and redaction of captured payloads From 2a4e7647187914fd8fdf00f12a0ffc3033280f31 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Wed, 23 Sep 2026 00:36:38 +1000 Subject: [PATCH 02/20] refactor(indexing): delete the workspace codesearch indexer The index built google/codesearch over the chat workspace on every daemon start, for the process lifetime, through a child-process helper and a SQLite metadata DB -- and grep consulted it only for searches confined to the workspace root, which the analysed sessions showed was a small fraction of searches. The compute was continuous; the benefit was not. Removes internal/indexing, the workspace-codesearch helper subcommand, the [indexing] config section, and the GrepIndex accelerator seam from the grep tool, which returns to its direct walk. google/codesearch leaves go.mod with it. --- Taskfile.yml | 10 +- cmd/archied/main.go | 6 - config.example.toml | 7 - deployments/README.md | 18 + docs/architecture/configuration.md | 6 +- go.mod | 17 +- go.sum | 46 +- internal/app/archied/bootstrap.go | 36 +- internal/app/archied/codesearch.go | 121 ---- internal/app/archied/codesearch_test.go | 85 --- internal/config/config.go | 2 - internal/config/indexer.go | 29 - internal/indexing/codesearch.go | 557 ------------------ internal/indexing/codesearch_test.go | 224 ------- .../infrastructure/configuration/defaults.go | 1 - .../configuration/loader_test.go | 49 -- .../configuration/tomlwrite/tomlwrite_test.go | 2 +- internal/releaseupdate/install_script_test.go | 5 +- internal/tools/builtin/builtin.go | 6 +- internal/tools/builtin/grep.go | 150 +---- internal/tools/builtin/grep_test.go | 120 +--- internal/tools/builtin/pathutil_test.go | 2 +- internal/tools/provider/builtin/provider.go | 25 +- .../tools/provider/builtin/provider_test.go | 73 --- scripts/archie-update-install | 5 +- 25 files changed, 139 insertions(+), 1463 deletions(-) delete mode 100644 internal/app/archied/codesearch.go delete mode 100644 internal/app/archied/codesearch_test.go delete mode 100644 internal/config/indexer.go delete mode 100644 internal/indexing/codesearch.go delete mode 100644 internal/indexing/codesearch_test.go diff --git a/Taskfile.yml b/Taskfile.yml index d059532cf..185dd0381 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -350,6 +350,10 @@ tasks: archie-gateway and archie-ui dial it -- none owns the DB in-process (docs/prds/state-store-contract.md §12 step 7) + * state admin read-only PocketBase surface over archie.db on + 127.0.0.1:8686 (view collections only; the gRPC + contract stays the only writer). Served by the + archie-state-store process, not a sixth one. * archie-ui serves the dashboard on 127.0.0.1:8484 from the embedded dist (deployments/dev.toml). After the UI cutover (archie-core-8cda.5.4) archied runs no @@ -380,8 +384,12 @@ tasks: dev:state-store: desc: archie-state-store only, no live-reload (prefer `task dev`) + # -admin-addr serves the read-only operator dashboard over the same + # archie.db this process already owns. It is a flag rather than a config + # key for the same reason -ready-addr is: the address is a property of + # how this process was launched, not of the deployment's config document. cmds: - - go run ./cmd/archie-state-store -config-overlay deployments/dev.toml + - go run ./cmd/archie-state-store -config-overlay deployments/dev.toml -admin-addr 127.0.0.1:8686 dev:gateway: desc: archie-gateway only, no live-reload (prefer `task dev`) diff --git a/cmd/archied/main.go b/cmd/archied/main.go index 003236493..a12379d30 100644 --- a/cmd/archied/main.go +++ b/cmd/archied/main.go @@ -15,11 +15,5 @@ func main() { if archied.IsSetupArgs(args) { os.Exit(archied.RunSetup(args[1:], os.Stdin, os.Stdout, os.Stderr)) } - // The codesearch helper is dispatched before the daemon's flags are - // parsed: it is this same binary re-invoked as a short-lived child by - // internal/indexing, and it must not touch config, the store or NATS. - if archied.IsCodesearchHelperArgs(args) { - os.Exit(archied.RunCodesearchHelper(args[1:], os.Stdout)) - } os.Exit(archied.Run()) } diff --git a/config.example.toml b/config.example.toml index 49506a1a7..bd7aa71d3 100644 --- a/config.example.toml +++ b/config.example.toml @@ -227,13 +227,6 @@ class = "ollama" # work_dir = "/path/to/workspace" # parallel_tool_calls = false -# Workspace code-search index. Paths default under work_dir when unset. -[indexing] -# Code-search index files (default: /indexes). -# index_dir = "/var/lib/archie/indexes" -# Per-index lifecycle metadata DB (default: /workspace-indexes.db). -# db_path = "/var/lib/archie/workspace-indexes.db" - # archied's own liveness surface, always on. GET /healthz and /health answer # 200 once the daemon is serving (503 while it is still booting), and # /health/detailed reports the readiness probes. This is what the update diff --git a/deployments/README.md b/deployments/README.md index 24c9c4d20..34afd05c1 100644 --- a/deployments/README.md +++ b/deployments/README.md @@ -124,3 +124,21 @@ indicates dual-store ownership (`docs/prds/state-store-contract.md` §12 step 7/ The readiness probe is the single-writer check the runbook has against a degraded store: a `503` from `/health/detailed` means `archied` should be started only after the store is healthy. + +### Inspecting the store + +`-admin-addr` starts a read-only operator dashboard on the same process, over +the file it already owns: + +```bash +archie-state-store -admin-addr 127.0.0.1:8686 # dashboard on http://127.0.0.1:8686/_/ +``` + +It co-tenants on `-tasks.sqlite` rather than opening a second +database, so it does not disturb the single-writer rule above: it runs inside +the process that holds the ownership claim, and every archie table is exposed +as a read-only view collection. The routes that could write past those views +(`POST /api/sql`, `/api/backups`) are refused, so the gRPC contract stays the +only writer and `archie-state-store backup` stays the only snapshot path. + +Leave the flag off and no dashboard listener is started. diff --git a/docs/architecture/configuration.md b/docs/architecture/configuration.md index 084dd7a75..a0e832bec 100644 --- a/docs/architecture/configuration.md +++ b/docs/architecture/configuration.md @@ -58,7 +58,7 @@ survive as a shared package, configuration domain, or renamed global model. Its loading behaviour moves to `internal/infrastructure/configuration`. Existing types and methods are reassigned by behaviour. Repository execution policy, dispatch policy, retry policy, ecosystem defaults, tool policy, memory settings, -indexing settings, provider settings, and channel settings move to their +provider settings, and channel settings move to their respective owners. ## Repository settings @@ -163,8 +163,7 @@ existing structure combines different owners. | webhook target | `Notify` | Notification adapter instance settings. | | web listen address | `Web` | Web transport settings. | | `listen` under `[services.state]` and `[services.gateway]` | `ServiceConnection` | The bind address of the process that owns the service. Read only by that process, and defaulted to the address its clients dial so the pair cannot drift. | -| index directory and database path | `Indexing` | Index infrastructure settings, derived from storage settings unless explicitly overridden. | -| forge, dispatch, models, providers, containers, memory, tools, indexing and repositories | `IdentityConfig` | Existing composite duplicates unrelated settings. Replace it with associations from an `IdentityID` to independently owned configured instances. | +| forge, dispatch, models, providers, containers, memory, tools and repositories | `IdentityConfig` | Existing composite duplicates unrelated settings. Replace it with associations from an `IdentityID` to independently owned configured instances. | | owner, name, base, gate, protection, ecosystem, preflight, storage, retry and concurrency | `Repo` | Split across work intake, workspace, workflow execution, policy, gate, and scheduling as described in Repository settings. | | `TaskConfig`, `TaskForge` | worker snapshot | Replace with the minimum immutable, versioned WorkflowExecution input assembled by the application. It is not a second configuration model. | | `Extra` and arbitrary maps | root/provider config | Allowed only at an extension boundary whose owning plugin/provider supplies schema and validation. Core configuration never consumes an untyped catch-all. | @@ -187,7 +186,6 @@ decision; exact setting names are implementation details. | NATS worker consumer | stream `ARCHIE_TASKS`, consumer `archie-agent`, deduplication `2m`, poll `5s`, acknowledgement wait `30m`, deliveries `3`, inactive threshold `24h`, queue group and retry delay `1s` | Worker transport settings and delivery/retry policy. Shared stream defaults must have one owner rather than two copies. | | Worker readiness | readiness timeout `20s`, retry delay `250ms` | Worker lifecycle and retry policy. | | Execution/RPC | agent execution `30m`, task-run RPC `60s`, worktree RPC `15m` | Per-operation timeout policy. | -| Indexing | build timeout `10m`, metadata timeout `5s` | Index operation policy. | | Plugin/tool cleanup | plugin rollback `10s`, tool-provider cleanup `10s`, MCP-provider cleanup `10s` | Lifecycle shutdown/rollback policy owned by each engine, with a shared policy vocabulary where appropriate. | | Memory read path | per-scope record limit `20`, rendered `` block byte cap `8192` | Memory read policy owned by the engine family (`internal/domain/memory`). | | Built-in memory store | per-file size limit `100KiB`, query result limit `100` | Split into memory resource policy, input policy, and presentation settings. | diff --git a/go.mod b/go.mod index 86462a16c..a5f4658a4 100644 --- a/go.mod +++ b/go.mod @@ -9,27 +9,36 @@ require ( github.com/ProtonMail/go-crypto v1.4.1 // indirect github.com/antithesishq/antithesis-sdk-go v0.8.0 // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect + github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudflare/circl v1.6.5 // indirect github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/davidmz/go-pageant v1.0.2 // indirect + github.com/disintegration/imaging v1.6.2 // indirect github.com/distribution/reference v0.6.0 // indirect github.com/docker/go-connections v0.8.1 // indirect github.com/docker/go-units v0.5.0 // indirect + github.com/domodwyer/mailyak/v3 v3.6.2 // indirect github.com/dustin/go-humanize v1.1.0 // indirect github.com/emirpasic/gods v1.18.1 // indirect + github.com/fatih/color v1.19.0 // indirect github.com/felixge/httpsnoop v1.1.0 // indirect + github.com/fsnotify/fsnotify v1.10.1 // indirect + github.com/gabriel-vasile/mimetype v1.4.15 // indirect + github.com/ganigeorgiev/fexpr v0.6.0 // indirect github.com/go-fed/httpsig v1.1.0 // indirect github.com/go-git/gcfg/v2 v2.0.2 // indirect github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/golang-jwt/jwt/v5 v5.3.1 // indirect github.com/google/go-querystring v1.2.0 // indirect github.com/google/go-tpm v0.9.8 // indirect github.com/hashicorp/go-version v1.9.0 // indirect github.com/kevinburke/ssh_config v1.6.0 // indirect github.com/klauspost/compress v1.20.0 // indirect github.com/klauspost/cpuid/v2 v2.4.0 // indirect + github.com/mattn/go-colorable v0.1.15 // indirect github.com/mattn/go-isatty v0.0.24 // indirect github.com/minio/highwayhash v1.0.4 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect @@ -41,8 +50,12 @@ require ( github.com/opencontainers/image-spec v1.1.1 // indirect github.com/pjbgf/sha1cd v0.6.0 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect + github.com/pocketbase/ozzo-validation/v4 v4.3.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/sergi/go-diff v1.4.0 // indirect + github.com/spf13/cast v1.10.0 // indirect + github.com/spf13/cobra v1.10.2 // indirect + github.com/spf13/pflag v1.0.10 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 // indirect go.opentelemetry.io/otel v1.46.0 // indirect @@ -51,6 +64,7 @@ require ( go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/crypto v0.57.0 // indirect golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba // indirect + golang.org/x/image v0.46.0 // indirect golang.org/x/sync v0.23.0 // indirect golang.org/x/sys v0.48.0 // indirect golang.org/x/text v0.42.0 // indirect @@ -71,13 +85,14 @@ require ( github.com/go-git/go-git/v6 v6.0.0-alpha.5 github.com/go-jose/go-jose/v4 v4.1.5 github.com/go-telegram/bot v1.27.0 - github.com/google/codesearch v1.3.0-rc.1 github.com/google/go-github/v78 v78.0.0 github.com/google/uuid v1.6.0 github.com/moby/moby/api v1.56.0 github.com/moby/moby/client v0.6.0 github.com/nats-io/nats-server/v2 v2.15.0 github.com/nats-io/nats.go v1.54.0 + github.com/pocketbase/dbx v1.12.0 + github.com/pocketbase/pocketbase v0.40.4 github.com/samcharles93/ai-sdk v0.1.39 github.com/traefik/yaegi v0.16.1 golang.org/x/net v0.59.0 diff --git a/go.sum b/go.sum index 7e0705ff4..deabe5645 100644 --- a/go.sum +++ b/go.sum @@ -25,6 +25,9 @@ github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYW github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/bwesterb/go-ristretto v1.2.4/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -38,17 +41,22 @@ github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmC github.com/containerd/typeurl/v2 v2.2.0/go.mod h1:8XOOxnyatxSWuG8OfsZXVnAF4iZfedjS/8UHSPJnX4g= github.com/coreos/go-oidc/v3 v3.21.0 h1:wZo4Q9Pum8dYEj0eMUPrqR+kvuGkeUplbLpNCkBqoWM= github.com/coreos/go-oidc/v3 v3.21.0/go.mod h1:DYCf24+ncYi+XkIH97GY1+dqoRlbaSI26KVTCI9SrY4= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davidmz/go-pageant v1.0.2 h1:bPblRCh5jGU+Uptpz6LgMZGD5hJoOt7otgT454WvHn0= github.com/davidmz/go-pageant v1.0.2/go.mod h1:P2EDDnMqIwG5Rrp05dTRITj9z2zpGcD9efWSkTNKLIE= +github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= +github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/docker/go-connections v0.8.1 h1:JibmG5hULs5qXSr/cp/w3Pw5fZuStt4MOHMUExb29/M= github.com/docker/go-connections v0.8.1/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/domodwyer/mailyak/v3 v3.6.2 h1:x3tGMsyFhTCaxp6ycgR0FE/bu5QiNp+hetUuCOBXMn8= +github.com/domodwyer/mailyak/v3 v3.6.2/go.mod h1:lOm/u9CyCVWHeaAmHIdF4RiKVxKUT/H5XX10lIKAL6c= github.com/dustin/go-humanize v1.1.0 h1:dbKTrvD0klcbBV/h4AWJdMuZogJACoMlvWIWZ5b2xWg= github.com/dustin/go-humanize v1.1.0/go.mod h1:hc1CvRkJMsgxqjmjMQF3QNRAZBwY8AXBAzKYoSX9sFI= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= @@ -57,8 +65,16 @@ github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9 github.com/envoyproxy/go-control-plane/envoy v1.39.0/go.mod h1:5e4ylfTZO723MEEFsCpSW4ZEBWR8mwkEyXfwJBTCZ9c= github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= +github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w= +github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE= github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= +github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= +github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= +github.com/gabriel-vasile/mimetype v1.4.15 h1:05iP/CYtZ/w455R/KZM6rZ5ieAdh99UPtd+d3YzLmaI= +github.com/gabriel-vasile/mimetype v1.4.15/go.mod h1:azpTcoLcDZRNgFou5j+APrqQx9HqVPWa6ijYQIIVswQ= +github.com/ganigeorgiev/fexpr v0.6.0 h1:Fza3O/QMBKEudUvxV862qe6GjxM60GJjjKytdp+VQus= +github.com/ganigeorgiev/fexpr v0.6.0/go.mod h1:RyGiGqmeXhEQ6+mlGdnUleLHgtzzu/VGO2WtJkF5drE= github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI= @@ -81,14 +97,16 @@ github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ4 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= +github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-telegram/bot v1.27.0 h1:PG9yBDzM8O26oUerE9SZwWlHon4RpH8WcNslZ7CEZmE= github.com/go-telegram/bot v1.27.0/go.mod h1:i2TRs7fXWIeaceF3z7KzsMt/he0TwkVC680mvdTFYeM= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/glog v1.2.5/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/codesearch v1.3.0-rc.1 h1:vSFGuF5wS3sjimuLzqPG+PGjWQ9VvEtPFbxdZD7YuuQ= -github.com/google/codesearch v1.3.0-rc.1/go.mod h1:hKdpx7wOsvk2WLT/D6NLPxFeLT8jKCehZu21JpA2rIY= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= @@ -110,6 +128,7 @@ github.com/hashicorp/go-version v1.9.0 h1:CeOIz6k+LoN3qX9Z0tyQrPtiB1DFYRPfCIBtaX github.com/hashicorp/go-version v1.9.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/kevinburke/ssh_config v1.6.0 h1:J1FBfmuVosPHf5GRdltRLhPJtJpTlMdKTBjRgTaQBFY= github.com/kevinburke/ssh_config v1.6.0/go.mod h1:q2RIzfka+BXARoNexmF9gkxEX7DmvbW9P4hIVx2Kg4M= github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= @@ -123,6 +142,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY= +github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= @@ -156,17 +177,31 @@ github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmd github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pocketbase/dbx v1.12.0 h1:/oLErM+A0b4xI0PWTGPqSDVjzix48PqI/bng2l0PzoA= +github.com/pocketbase/dbx v1.12.0/go.mod h1:xXRCIAKTHMgUCyCKZm55pUOdvFziJjQfXaWKhu2vhMs= +github.com/pocketbase/ozzo-validation/v4 v4.3.0 h1:uKBDVma7bZqgR2a6AwE+k9hkuDFfiZMpBHQdZ1z3iQs= +github.com/pocketbase/ozzo-validation/v4 v4.3.0/go.mod h1:6XNjSTw/Jb2F8LOkKO3oyzIWExbrGiYoS4uVxVwz90g= +github.com/pocketbase/pocketbase v0.40.4 h1:0SvSUreR3NhUMCs9LchE59oEG53efZ3cKiMGyAGBN9U= +github.com/pocketbase/pocketbase v0.40.4/go.mod h1:2mU+80FLiY1fb13WZRg8Xx/lKg4nTjgKVJMigyRH6k0= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/samcharles93/ai-sdk v0.1.39 h1:gbNPYeA27MifReCruBiqMgb4fFSjCDi1Zv57a9bo0Vo= github.com/samcharles93/ai-sdk v0.1.39/go.mod h1:u4zrQ9/Q58L5mNEut4bNQ1ks+RCKq71LN5EDz2N/0Zk= github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= +github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spiffe/go-spiffe/v2 v2.8.1/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -189,6 +224,7 @@ go.opentelemetry.io/otel/sdk/metric v1.46.0 h1:0piZ26EG4RBfebb2jhDH6ERCYHoVWduc3 go.opentelemetry.io/otel/sdk/metric v1.46.0/go.mod h1:I1PbKrdVc8Qu8HYVDNtqVIwLwjNrhsV/uFuxfwg8mO4= go.opentelemetry.io/otel/trace v1.46.0 h1:OULy7ccdJnZtJ0UDYFOIGaCmiWzJ8Vi2G/Rsu60qs1c= go.opentelemetry.io/otel/trace v1.46.0/go.mod h1:J7GAXweO77XSFkB/rmAqk9D6ihszhFjLU+d9WuUxDLI= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -198,9 +234,13 @@ golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba h1:Ck8QetSgk912qxWLMCKxd0in+aiyBQyDSMae6e/xmpU= golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba/go.mod h1:50RgIsmK7OwqzTTeqcSXQW8SswW0o8fRcDxmqGluJ8E= +golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.46.0 h1:b1+oYj0Jbp6K5MDT4i4/eZpYlk3V8SJhhDKh6LBHAyQ= +golang.org/x/image v0.46.0/go.mod h1:3B3W05VGVQyuXucLINLjXKrqISASfi4Xj+iCVkLMwew= golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= @@ -221,6 +261,7 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= @@ -234,6 +275,7 @@ golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated/go.mod h1:RVAQXBGN gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/api v0.293.0/go.mod h1:6n5tjEB1gzwniZTepZ0g5u+wM7Bof5GeULCx/zh8ZE0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto/googleapis/api v0.0.0-20260918162117-cecb64721679 h1:FEp7JNE32DTAwbnI/ixagnmj7Xm1eTONofGEUXFjZ4w= google.golang.org/genproto/googleapis/api v0.0.0-20260918162117-cecb64721679/go.mod h1:52bV8FLAQ9Qmcqaq9ECLmuEHZthk+6OPV45aKBBrsNw= google.golang.org/genproto/googleapis/api v0.0.0-20260921155816-b14227669459 h1:GS9OIt/j7c8bvBjYNgnKQysVfmV7e4jM0H8ZK95G4t8= diff --git a/internal/app/archied/bootstrap.go b/internal/app/archied/bootstrap.go index f868dff9a..72e526297 100644 --- a/internal/app/archied/bootstrap.go +++ b/internal/app/archied/bootstrap.go @@ -45,7 +45,6 @@ import ( "github.com/samcharles93/archie-core/internal/forge" forgewebhook "github.com/samcharles93/archie-core/internal/forge/webhook" "github.com/samcharles93/archie-core/internal/gateway" - "github.com/samcharles93/archie-core/internal/indexing" "github.com/samcharles93/archie-core/internal/infrastructure/configuration" infraembedding "github.com/samcharles93/archie-core/internal/infrastructure/embedding" "github.com/samcharles93/archie-core/internal/infrastructure/eventbus/nats" @@ -65,7 +64,6 @@ import ( "github.com/samcharles93/archie-core/internal/skill" "github.com/samcharles93/archie-core/internal/storage" "github.com/samcharles93/archie-core/internal/tools" - toolsbuiltin "github.com/samcharles93/archie-core/internal/tools/builtin" "github.com/samcharles93/archie-core/internal/tools/minimax" toolprovider "github.com/samcharles93/archie-core/internal/tools/provider" builtintoolprovider "github.com/samcharles93/archie-core/internal/tools/provider/builtin" @@ -1115,37 +1113,6 @@ func (b *boot) setupGuardrails() { ) } -// workspaceIndex builds the optional codesearch index that narrows grep's -// candidate files. It returns a slice so a failure is simply "no index" -// rather than a typed-nil reaching the provider. -// -// Every failure degrades instead of propagating: grep falls back to its -// direct walk, which is what it did before this was wired at all. An -// accelerator that can refuse to build must never stop the daemon starting. -// -// The context is detached from cancellation but keeps its values, because -// NewManager starts an asynchronous build whose lifetime is the process -// while the gateway path's boot context is cancelled once startup returns. -// Inheriting cancellation would kill the build mid-flight and leave grep -// permanently unindexed with nothing to show why; using Background instead -// would silently drop any values the caller's context carries. -func (b *boot) workspaceIndex(ctx context.Context, workspace string) []toolsbuiltin.GrepIndex { - cfg, log := b.cfg, b.log - idxCfg := cfg.Indexing.WithDefaults(cfg.WorkDir) - manager, err := indexing.NewManager( - context.WithoutCancel(ctx), - indexing.Config{IndexDir: idxCfg.IndexDir, DBPath: idxCfg.DBPath, Log: log}, - workspace, - ) - if err != nil { - log.Warn("workspace index unavailable; grep will search unindexed", - "workspace", workspace, "index_dir", idxCfg.IndexDir, "error", err) - return nil - } - log.Info("workspace index enabled", "workspace", workspace, "index_dir", idxCfg.IndexDir) - return []toolsbuiltin.GrepIndex{manager} -} - // registerTools registers the tool providers with a lifecycle: workspace // file/shell tools and optional MCP servers. Memory tools // (memory_create/update/delete/list) are not registered here -- they are @@ -1159,8 +1126,7 @@ func (b *boot) registerTools(ctx context.Context) error { // deliberate choice rather than a default. if workspace := cfg.Chat.Workspace; workspace != "" { unrestricted := cfg.Chat.UnrestrictedFilesystem - index := b.workspaceIndex(ctx, workspace) - if err := b.providerRegistry.Register(builtintoolprovider.New(workspace, unrestricted, index...)); err != nil { + if err := b.providerRegistry.Register(builtintoolprovider.New(workspace, unrestricted)); err != nil { log.Error("workspace tool provider registration failed", "err", err) return err } diff --git a/internal/app/archied/codesearch.go b/internal/app/archied/codesearch.go deleted file mode 100644 index da03e84c2..000000000 --- a/internal/app/archied/codesearch.go +++ /dev/null @@ -1,121 +0,0 @@ -package archied - -import ( - "context" - "encoding/json" - "flag" - "fmt" - "io" - "os" - - "github.com/samcharles93/archie-core/internal/indexing" -) - -// flagErrorOutput receives helper diagnostics. It is a variable so tests can -// capture them, and it is deliberately not stdout: the parent parses stdout -// as JSON, so a stray usage message there would corrupt the result. -var flagErrorOutput io.Writer = os.Stderr - -// codesearchHelperCommand is the argv[1] token that switches archied out of -// daemon mode and into the codesearch helper. -// -// indexing.Manager deliberately runs the google/codesearch library in a child -// process: upstream calls fatal exits on a corrupt index and leaks mmapped -// regions, neither of which may be allowed to take down a long-lived daemon. -// The child it spawns is this binary, so this name is a contract with -// internal/indexing -- renaming either side silently disables indexed search -// and falls the agent back to unindexed grep. -const codesearchHelperCommand = "workspace-codesearch" - -// IsCodesearchHelperArgs reports whether args (os.Args[1:]) selects the -// helper rather than the daemon. -func IsCodesearchHelperArgs(args []string) bool { - return len(args) > 0 && args[0] == codesearchHelperCommand -} - -// RunCodesearchHelper executes one helper subcommand and returns a process -// exit code. Output is the machine-readable result the parent decodes; all -// diagnostics go to stderr so they cannot corrupt it. -func RunCodesearchHelper(args []string, out io.Writer) int { - if len(args) == 0 { - helperErrorf("%s: expected a subcommand (build|candidates)\n", codesearchHelperCommand) - return 2 - } - - ctx := context.Background() - switch args[0] { - case "build": - fs := newHelperFlagSet("build") - root := fs.String("root", "", "workspace root to index") - index := fs.String("index", "", "path of the index sidecar to write") - if err := fs.Parse(args[1:]); err != nil { - return 2 - } - if *root == "" || *index == "" { - helperErrorf("build: --root and --index are required\n") - return 2 - } - if err := indexing.BuildIndex(ctx, *root, *index); err != nil { - helperErrorf("build: %v\n", err) - return 1 - } - return 0 - - case "candidates": - return runCandidates(args[1:], out) - - default: - helperErrorf("%s: unknown subcommand %q\n", codesearchHelperCommand, args[0]) - return 2 - } -} - -func newHelperFlagSet(name string) *flag.FlagSet { - fs := flag.NewFlagSet(name, flag.ContinueOnError) - fs.SetOutput(flagErrorOutput) - return fs -} - -// runCandidates handles the "candidates" subcommand, returning candidate -// file paths as a JSON array on stdout. -func runCandidates(args []string, out io.Writer) int { - fs := newHelperFlagSet("candidates") - index := fs.String("index", "", "path of the index sidecar to query") - pattern := fs.String("pattern", "", "search pattern") - literal := fs.Bool("literal", false, "treat the pattern as a literal string") - caseSensitive := fs.Bool("case-sensitive", false, "match case sensitively") - if err := fs.Parse(args); err != nil { - return 2 - } - if *index == "" || *pattern == "" { - helperErrorf("candidates: --index and --pattern are required\n") - return 2 - } - files, err := indexing.IndexCandidates(*index, *pattern, *literal, *caseSensitive) - if err != nil { - helperErrorf("candidates: %v\n", err) - return 1 - } - // The parent decodes a JSON array and treats any decode failure as - // "no index", so a nil slice must still marshal as [] rather than null. - if files == nil { - files = []string{} - } - encoded, err := json.Marshal(files) - if err != nil { - helperErrorf("candidates: encode result: %v\n", err) - return 1 - } - if _, err := out.Write(encoded); err != nil { - helperErrorf("candidates: write result: %v\n", err) - return 1 - } - return 0 -} - -// helperErrorf writes a diagnostic to stderr. The write result is ignored -// deliberately: if stderr is gone there is nowhere left to report it, and the -// exit code already carries the outcome the parent acts on. -func helperErrorf(format string, args ...any) { - _, _ = fmt.Fprintf(flagErrorOutput, format, args...) -} diff --git a/internal/app/archied/codesearch_test.go b/internal/app/archied/codesearch_test.go deleted file mode 100644 index d28c85eff..000000000 --- a/internal/app/archied/codesearch_test.go +++ /dev/null @@ -1,85 +0,0 @@ -package archied - -import ( - "encoding/json" - "os" - "path/filepath" - "slices" - "strings" - "testing" -) - -func TestRunCodesearchHelper(t *testing.T) { - root := t.TempDir() - if err := os.WriteFile(filepath.Join(root, "a.txt"), []byte("alpha needle omega\n"), 0o600); err != nil { - t.Fatalf("write fixture: %v", err) - } - if err := os.WriteFile(filepath.Join(root, "b.txt"), []byte("nothing here\n"), 0o600); err != nil { - t.Fatalf("write fixture: %v", err) - } - indexPath := filepath.Join(t.TempDir(), "workspace.csearch") - - t.Run("build writes an index sidecar", func(t *testing.T) { - var out strings.Builder - code := RunCodesearchHelper([]string{"build", "--root", root, "--index", indexPath}, &out) - if code != 0 { - t.Fatalf("build exit code = %d, want 0", code) - } - if _, err := os.Stat(indexPath); err != nil { - t.Fatalf("index sidecar not written: %v", err) - } - }) - - t.Run("candidates emits the JSON contract the manager decodes", func(t *testing.T) { - var out strings.Builder - code := RunCodesearchHelper([]string{ - "candidates", "--index", indexPath, "--pattern", "needle", - }, &out) - if code != 0 { - t.Fatalf("candidates exit code = %d, want 0", code) - } - var files []string - if err := json.Unmarshal([]byte(out.String()), &files); err != nil { - t.Fatalf("decode candidates output %q: %v", out.String(), err) - } - if !slices.ContainsFunc(files, func(p string) bool { - return strings.HasSuffix(p, "a.txt") - }) { - t.Errorf("candidates = %v, want the file containing the needle", files) - } - }) - - invalid := []struct { - name string - args []string - }{ - {name: "no subcommand", args: nil}, - {name: "unknown subcommand", args: []string{"frobnicate"}}, - {name: "build without root", args: []string{"build", "--index", indexPath}}, - {name: "build without index", args: []string{"build", "--root", root}}, - {name: "candidates without index", args: []string{"candidates", "--pattern", "x"}}, - {name: "candidates without pattern", args: []string{"candidates", "--index", indexPath}}, - } - for _, tc := range invalid { - t.Run(tc.name+" is rejected", func(t *testing.T) { - var out strings.Builder - if code := RunCodesearchHelper(tc.args, &out); code == 0 { - t.Errorf("exit code = 0, want non-zero for %v", tc.args) - } - }) - } -} - -// The manager shells out to argv[0] with this exact subcommand name, so a -// rename on either side silently disables indexed search. -func TestCodesearchHelperSubcommandIsRecognised(t *testing.T) { - if !IsCodesearchHelperArgs([]string{codesearchHelperCommand, "build"}) { - t.Errorf("IsCodesearchHelperArgs() did not recognise %q", codesearchHelperCommand) - } - if IsCodesearchHelperArgs([]string{"-once"}) { - t.Error("IsCodesearchHelperArgs() claimed a normal daemon flag") - } - if IsCodesearchHelperArgs(nil) { - t.Error("IsCodesearchHelperArgs() claimed an empty argument list") - } -} diff --git a/internal/config/config.go b/internal/config/config.go index ff0af1788..4eeb1de13 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -533,8 +533,6 @@ type Config struct { // Tools holds MCP server and tool policy configuration (from config.tools.yaml). Tools ToolsConfig `toml:"tools" yaml:"tools"` - // Indexing locates the workspace codesearch index. - Indexing IndexingConfig `toml:"indexing" yaml:"indexing"` // Identities declares multi-identity configurations. When non-empty, // each identity runs its own poll loop with its own forge client, diff --git a/internal/config/indexer.go b/internal/config/indexer.go deleted file mode 100644 index 3bc863563..000000000 --- a/internal/config/indexer.go +++ /dev/null @@ -1,29 +0,0 @@ -package config - -import "path/filepath" - -// IndexingConfig locates the workspace codesearch index. -// -// The index is a large, fully rebuildable artifact, so both paths are -// settable to allow putting it on different storage from the task work tree. -type IndexingConfig struct { - // IndexDir holds the codesearch sidecar files. - IndexDir string `toml:"index_dir" yaml:"index_dir" json:"index_dir,omitempty"` - // DBPath is the SQLite database holding index lifecycle metadata. - DBPath string `toml:"db_path" yaml:"db_path" json:"db_path,omitempty"` -} - -// WithDefaults fills unset paths from the daemon's work directory. -// -// Deriving from workDir rather than a shared data home is deliberate: two -// archied instances run on one host, each with its own work_dir, so this -// keeps their indexes separate without either having to configure anything. -func (c IndexingConfig) WithDefaults(workDir string) IndexingConfig { - if c.IndexDir == "" { - c.IndexDir = filepath.Join(workDir, "indexes") - } - if c.DBPath == "" { - c.DBPath = filepath.Join(workDir, "workspace-indexes.db") - } - return c -} diff --git a/internal/indexing/codesearch.go b/internal/indexing/codesearch.go deleted file mode 100644 index adf8962b7..000000000 --- a/internal/indexing/codesearch.go +++ /dev/null @@ -1,557 +0,0 @@ -// Package indexing provides language-neutral workspace text indexing. -package indexing - -import ( - "context" - "crypto/sha256" - "database/sql" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "log/slog" - "os" - "os/exec" - "path/filepath" - "regexp" - "regexp/syntax" - "slices" - "strings" - "time" - - "github.com/google/codesearch/index" - "github.com/google/uuid" - _ "modernc.org/sqlite" -) - -const ( - // MaxCandidateFiles bounds helper output and grep argv construction. - MaxCandidateFiles = 2000 - maxCandidateArgBytes = 24 * 1024 - buildTimeout = 10 * time.Minute - metadataTimeout = 5 * time.Second -) - -// CandidateProvider returns files which may match a grep query. A false -// handled result tells the caller to use its normal unindexed search path. -type CandidateProvider interface { - Candidates(context.Context, string, bool, bool) ([]string, bool) -} - -// Manager owns the workspace index lifecycle. The codesearch implementation -// runs in a child process so upstream fatal exits and mmap lifetime cannot -// terminate or leak inside the interactive agent process. -type Manager struct { - root string - indexPath string - dbPath string - runner helperRunner - log *slog.Logger -} - -// logger is nil-safe: a Manager may be constructed as a struct literal -// (the tests do), so defaulting at the point of use rather than in the -// constructor is what actually holds. -func (m *Manager) logger() *slog.Logger { - if m == nil || m.log == nil { - return slog.Default() - } - return m.log -} - -type helperRunner interface { - Build(context.Context, string, string) error - Candidates(context.Context, string, string, bool, bool) ([]string, error) -} - -type processRunner struct{} - -// Config locates the index storage for a Manager. -// -// Archied resolves these in its config layer ([indexing] in config.toml, -// defaulted off work_dir by config.finalize) and passes them down. Defaulting -// deliberately does NOT live here so this package stays free of archie-core's -// config. -type Config struct { - // IndexDir holds the mmap-friendly codesearch sidecar files. - IndexDir string - // DBPath is the SQLite database holding index lifecycle metadata. - DBPath string - // Log receives index build failures. Without it a failed build is - // invisible: the manager degrades to unindexed search and the reason - // is only ever written to a database column nobody reads. - Log *slog.Logger -} - -// NewManager prepares lifecycle storage and starts an asynchronous atomic -// refresh. An existing index remains available while the refresh runs. -func NewManager(ctx context.Context, cfg Config, root string) (*Manager, error) { - if strings.TrimSpace(cfg.IndexDir) == "" { - return nil, fmt.Errorf("indexing: IndexDir is required") - } - if strings.TrimSpace(cfg.DBPath) == "" { - return nil, fmt.Errorf("indexing: DBPath is required") - } - absRoot, err := filepath.Abs(root) - if err != nil { - return nil, fmt.Errorf("resolve workspace root: %w", err) - } - if err := os.MkdirAll(cfg.IndexDir, 0o700); err != nil { - return nil, fmt.Errorf("create workspace index directory: %w", err) - } - m := &Manager{ - root: filepath.Clean(absRoot), - indexPath: filepath.Join(cfg.IndexDir, workspaceKey(absRoot)+".csearch"), - dbPath: cfg.DBPath, - runner: processRunner{}, - log: cfg.Log, - } - if err := m.ensureSchema(ctx); err != nil { - return nil, err - } - m.refreshAsync(ctx) - return m, nil -} - -// Candidates implements the tools.GrepIndex contract. -func (m *Manager) Candidates(ctx context.Context, pattern string, literal, caseSensitive bool) ([]string, bool) { - state, err := m.state(ctx) - if err != nil || state.IndexedAt.IsZero() { - return nil, false - } - if _, err := os.Stat(state.IndexPath); err != nil { - return nil, false - } - files, err := m.runner.Candidates(ctx, state.IndexPath, pattern, literal, caseSensitive) - if err != nil { - return nil, false - } - - // The sidecar is a snapshot. Include files changed or created since the - // build began so indexed search cannot miss live workspace edits. - merged, ok := m.mergeCandidateFiles(ctx, files, state) - if !ok { - return nil, false - } - slices.Sort(merged) - return merged, true -} - -// mergeCandidateFiles combines sidecar results with workspace files that -// were changed or created since the index was built. -func (m *Manager) mergeCandidateFiles(ctx context.Context, files []string, state indexState) ([]string, bool) { - workspaceFiles, err := WorkspaceFiles(ctx, m.root) - if err != nil { - return nil, false - } - changedFiles := changedGitFiles(ctx, m.root) - seen := make(map[string]struct{}, len(files)) - merged := make([]string, 0, len(files)) - argumentBytes := 0 - add := func(path string) { - path, ok := confinedWorkspaceFile(m.root, path) - if !ok { - return - } - if _, ok := seen[path]; ok { - return - } - seen[path] = struct{}{} - merged = append(merged, path) - argumentBytes += len(path) + 1 - } - for _, path := range files { - add(path) - } - for _, path := range workspaceFiles { - info, statErr := os.Stat(path) - _, changed := changedFiles[path] - if statErr == nil && (changed || !info.ModTime().Before(state.IndexedAt)) { - add(path) - } - } - if len(merged) == 0 || len(merged) > MaxCandidateFiles || argumentBytes > maxCandidateArgBytes { - return nil, false - } - return merged, true -} - -func changedGitFiles(ctx context.Context, root string) map[string]struct{} { - changed := map[string]struct{}{} - commands := [][]string{ - {"-C", root, "ls-files", "-m", "-o", "--exclude-standard", "-z"}, - {"-C", root, "diff", "--cached", "--name-only", "--diff-filter=ACMR", "-z"}, - } - for _, args := range commands { - out, err := exec.CommandContext(ctx, "git", args...).Output() - if err != nil { - return map[string]struct{}{} - } - for name := range strings.SplitSeq(string(out), "\x00") { - if name != "" { - path := filepath.Join(root, filepath.FromSlash(name)) - if confined, ok := confinedWorkspaceFile(root, path); ok { - changed[confined] = struct{}{} - } - } - } - } - return changed -} - -type indexState struct { - IndexPath string - IndexedAt time.Time -} - -func (m *Manager) refreshAsync(parent context.Context) { - go func() { - ctx, cancel := context.WithTimeout(parent, buildTimeout) - defer cancel() - startedAt := time.Now().UTC() - buildID := uuid.NewString() - previous, _ := m.state(ctx) - claimed, err := m.claimBuild(ctx, startedAt, buildID) - if err != nil { - // Degrading to unindexed search is safe but slow, and until - // this is logged the only record of why is a database column - // with no reader. - m.logger().Warn("workspace index build not claimed", "root", m.root, "error", err) - return - } - if !claimed { - // Another process holds the build claim; not an error. - return - } - finalPath := m.indexPath + "." + buildID - tmp := finalPath + ".tmp" - _ = os.Remove(tmp) - if err := m.runner.Build(ctx, m.root, tmp); err != nil { - _ = os.Remove(tmp) - m.logger().Warn("workspace index build failed; falling back to unindexed search", - "root", m.root, "error", err) - m.finishBuild(parent, buildID, "error", time.Time{}, "", err.Error()) - return - } - if err := os.Rename(tmp, finalPath); err != nil { - _ = os.Remove(tmp) - m.logger().Warn("workspace index could not be published", - "root", m.root, "error", err) - m.finishBuild(parent, buildID, "error", time.Time{}, "", err.Error()) - return - } - if !m.finishBuild(parent, buildID, "ready", startedAt, finalPath, "") { - _ = os.Remove(finalPath) - m.logger().Warn("workspace index build superseded before publication", "root", m.root) - return - } - m.logger().Info("workspace index ready", "root", m.root, "index", finalPath) - if previous.IndexPath != "" && previous.IndexPath != finalPath && - (previous.IndexPath == m.indexPath || strings.HasPrefix(previous.IndexPath, m.indexPath+".")) { - _ = os.Remove(previous.IndexPath) - } - }() -} - -func (m *Manager) finishBuild(parent context.Context, buildID, status string, indexedAt time.Time, indexPath, lastError string) bool { - ctx, cancel := context.WithTimeout(context.WithoutCancel(parent), metadataTimeout) - defer cancel() - db, err := openIndexDB(m.dbPath) - if err != nil { - return false - } - // Each helper opens its own single-connection handle and the write has - // already been committed by the time this runs, so a Close error carries - // no recoverable information. - defer func() { _ = db.Close() }() - indexed := "" - if !indexedAt.IsZero() { - indexed = indexedAt.Format(time.RFC3339Nano) - } - result, err := db.ExecContext(ctx, `UPDATE workspace_indexes SET - index_path=CASE WHEN ? = 'ready' THEN ? ELSE index_path END, - status=?, indexed_at=CASE WHEN ? = 'ready' THEN ? ELSE indexed_at END, - last_error=?, updated_at=?, generation=generation + CASE WHEN ? = 'ready' THEN 1 ELSE 0 END - WHERE root=? AND build_id=?`, status, indexPath, status, status, indexed, - lastError, time.Now().UTC().Format(time.RFC3339Nano), status, m.root, buildID) - if err != nil { - return false - } - rows, err := result.RowsAffected() - return err == nil && rows == 1 -} - -func (m *Manager) claimBuild(ctx context.Context, startedAt time.Time, buildID string) (bool, error) { - db, err := openIndexDB(m.dbPath) - if err != nil { - return false, err - } - // Each helper opens its own single-connection handle and the write has - // already been committed by the time this runs, so a Close error carries - // no recoverable information. - defer func() { _ = db.Close() }() - cutoff := startedAt.Add(-buildTimeout).Format(time.RFC3339Nano) - result, err := db.ExecContext(ctx, `INSERT INTO workspace_indexes (root, index_path, status, indexed_at, build_started_at, build_id, last_error, updated_at) - VALUES (?, ?, 'building', '', ?, ?, '', ?) - ON CONFLICT(root) DO UPDATE SET status='building', - build_started_at=excluded.build_started_at, build_id=excluded.build_id, last_error='', updated_at=excluded.updated_at - WHERE workspace_indexes.status != 'building' OR workspace_indexes.updated_at < ?`, - m.root, m.indexPath, startedAt.Format(time.RFC3339Nano), buildID, startedAt.Format(time.RFC3339Nano), cutoff) - if err != nil { - return false, fmt.Errorf("claim workspace index build: %w", err) - } - rows, err := result.RowsAffected() - if err != nil { - return false, fmt.Errorf("inspect workspace index build claim: %w", err) - } - return rows > 0, nil -} - -func (m *Manager) ensureSchema(ctx context.Context) error { - db, err := openIndexDB(m.dbPath) - if err != nil { - return err - } - // Each helper opens its own single-connection handle and the write has - // already been committed by the time this runs, so a Close error carries - // no recoverable information. - defer func() { _ = db.Close() }() - _, err = db.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS workspace_indexes ( - root TEXT PRIMARY KEY, - index_path TEXT NOT NULL, - status TEXT NOT NULL, - format_version TEXT NOT NULL DEFAULT 'codesearch-v2', - generation INTEGER NOT NULL DEFAULT 0, - indexed_at TEXT NOT NULL DEFAULT '', - build_started_at TEXT NOT NULL DEFAULT '', - build_id TEXT NOT NULL DEFAULT '', - last_error TEXT NOT NULL DEFAULT '', - updated_at TEXT NOT NULL - )`) - if err != nil { - return fmt.Errorf("create workspace index metadata: %w", err) - } - for _, migration := range []string{ - `ALTER TABLE workspace_indexes ADD COLUMN format_version TEXT NOT NULL DEFAULT 'codesearch-v2'`, - `ALTER TABLE workspace_indexes ADD COLUMN generation INTEGER NOT NULL DEFAULT 0`, - `ALTER TABLE workspace_indexes ADD COLUMN build_started_at TEXT NOT NULL DEFAULT ''`, - `ALTER TABLE workspace_indexes ADD COLUMN build_id TEXT NOT NULL DEFAULT ''`, - } { - if _, migrationErr := db.ExecContext(ctx, migration); migrationErr != nil && !strings.Contains(migrationErr.Error(), "duplicate column name") { - return fmt.Errorf("migrate workspace index metadata: %w", migrationErr) - } - } - return nil -} - -func (m *Manager) state(ctx context.Context) (indexState, error) { - db, err := openIndexDB(m.dbPath) - if err != nil { - return indexState{}, err - } - // Each helper opens its own single-connection handle and the write has - // already been committed by the time this runs, so a Close error carries - // no recoverable information. - defer func() { _ = db.Close() }() - var indexed, indexPath string - err = db.QueryRowContext(ctx, `SELECT index_path, indexed_at FROM workspace_indexes WHERE root = ?`, m.root).Scan(&indexPath, &indexed) - if errors.Is(err, sql.ErrNoRows) { - return indexState{}, nil - } - if err != nil { - return indexState{}, fmt.Errorf("read workspace index metadata: %w", err) - } - at, _ := time.Parse(time.RFC3339Nano, indexed) - return indexState{IndexPath: indexPath, IndexedAt: at}, nil -} - -func openIndexDB(path string) (*sql.DB, error) { - // modernc.org/sqlite only applies pragmas passed via repeated _pragma - // params; unrecognized keys like _journal_mode/_busy_timeout are - // silently ignored, which previously left the busy timeout at 0 and - // caused SQLITE_BUSY on any lock contention instead of a bounded wait. - db, err := sql.Open("sqlite", path+"?_pragma=busy_timeout(5000)&_pragma=journal_mode(WAL)") - if err != nil { - return nil, fmt.Errorf("open workspace index metadata: %w", err) - } - db.SetMaxOpenConns(1) - return db, nil -} - -func workspaceKey(root string) string { - sum := sha256.Sum256([]byte(filepath.Clean(root))) - return hex.EncodeToString(sum[:12]) -} - -func (processRunner) Build(ctx context.Context, root, target string) error { - _, err := runHelper(ctx, "build", "--root", root, "--index", target) - return err -} - -func (processRunner) Candidates(ctx context.Context, indexPath, pattern string, literal, caseSensitive bool) ([]string, error) { - args := []string{"candidates", "--index", indexPath, "--pattern", pattern} - if literal { - args = append(args, "--literal") - } - if caseSensitive { - args = append(args, "--case-sensitive") - } - out, err := runHelper(ctx, args...) - if err != nil { - return nil, err - } - var files []string - if err := json.Unmarshal(out, &files); err != nil { - return nil, fmt.Errorf("decode codesearch candidates: %w", err) - } - return files, nil -} - -func runHelper(ctx context.Context, args ...string) ([]byte, error) { - executable, err := os.Executable() - if err != nil { - return nil, fmt.Errorf("locate archied executable: %w", err) - } - cmd := exec.CommandContext(ctx, executable, append([]string{"workspace-codesearch"}, args...)...) - out, err := cmd.Output() - if err != nil { - if exitErr, ok := errors.AsType[*exec.ExitError](err); ok { - return nil, fmt.Errorf("codesearch helper: %s", strings.TrimSpace(string(exitErr.Stderr))) - } - return nil, fmt.Errorf("run codesearch helper: %w", err) - } - return out, nil -} - -// BuildIndex creates a codesearch sidecar for root. It is called only inside -// the isolated helper process. -func BuildIndex(ctx context.Context, root, target string) error { - files, err := WorkspaceFiles(ctx, root) - if err != nil { - return err - } - w := index.Create(target) - w.AddRoots([]index.Path{index.MakePath(root)}) - for _, path := range files { - if err := w.AddFile(path); err != nil { - continue - } - } - w.Flush() - return nil -} - -// IndexCandidates returns the conservative trigram candidate set. It is -// called only inside the isolated helper process. -func IndexCandidates(indexPath, pattern string, literal, caseSensitive bool) (files []string, err error) { - defer func() { - if recovered := recover(); recovered != nil { - files = nil - err = fmt.Errorf("open codesearch index: %v", recovered) - } - }() - if literal { - pattern = regexp.QuoteMeta(pattern) - } - if !caseSensitive && !hasUppercase(pattern) { - pattern = "(?i:" + pattern + ")" - } - re, err := syntax.Parse(pattern, syntax.Perl) - if err != nil { - return nil, fmt.Errorf("parse search pattern: %w", err) - } - ix := index.Open(indexPath) - ids := ix.PostingQuery(index.RegexpQuery(re)) - files = make([]string, 0, len(ids)) - for _, id := range ids { - files = append(files, ix.Name(id).String()) - } - return files, nil -} - -// WorkspaceFiles lists searchable workspace files, preferring Git's ignore -// semantics and falling back to a conservative filesystem walk. -func WorkspaceFiles(ctx context.Context, root string) ([]string, error) { - cmd := exec.CommandContext(ctx, "git", "-C", root, "ls-files", "-co", "--exclude-standard", "-z") - out, err := cmd.Output() - if err == nil { - parts := strings.Split(string(out), "\x00") - files := make([]string, 0, len(parts)) - for _, part := range parts { - if part != "" { - path := filepath.Join(root, filepath.FromSlash(part)) - if confined, ok := confinedWorkspaceFile(root, path); ok { - files = append(files, confined) - } - } - } - slices.Sort(files) - return files, nil - } - - var files []string - err = filepath.WalkDir(root, func(path string, entry os.DirEntry, walkErr error) error { - if walkErr != nil { - return nil //nolint:nilerr // an unreadable entry skips that path and continues the walk; a best-effort index must not abort on one bad file - } - if path != root && strings.HasPrefix(entry.Name(), ".") { - if entry.IsDir() { - return filepath.SkipDir - } - return nil - } - if !entry.IsDir() && entry.Type().IsRegular() { - if confined, ok := confinedWorkspaceFile(root, path); ok { - files = append(files, confined) - } - } - return nil - }) - if err != nil { - return nil, fmt.Errorf("walk workspace files: %w", err) - } - slices.Sort(files) - return files, nil -} - -func confinedWorkspaceFile(root, path string) (string, bool) { - absRoot, err := filepath.Abs(root) - if err != nil { - return "", false - } - absPath, err := filepath.Abs(path) - if err != nil { - return "", false - } - // Reject a symlink at the final component before resolving parent links. - info, err := os.Lstat(absPath) - if err != nil || !info.Mode().IsRegular() { - return "", false - } - resolvedRoot, err := filepath.EvalSymlinks(absRoot) - if err != nil { - return "", false - } - resolvedPath, err := filepath.EvalSymlinks(absPath) - if err != nil { - return "", false - } - rel, err := filepath.Rel(resolvedRoot, resolvedPath) - if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { - return "", false - } - resolvedInfo, err := os.Lstat(resolvedPath) - if err != nil || !resolvedInfo.Mode().IsRegular() { - return "", false - } - return filepath.Clean(resolvedPath), true -} - -func hasUppercase(value string) bool { - for _, r := range value { - if r >= 'A' && r <= 'Z' { - return true - } - } - return false -} diff --git a/internal/indexing/codesearch_test.go b/internal/indexing/codesearch_test.go deleted file mode 100644 index c02cf64cb..000000000 --- a/internal/indexing/codesearch_test.go +++ /dev/null @@ -1,224 +0,0 @@ -package indexing - -import ( - "context" - "os" - "os/exec" - "path/filepath" - "slices" - "testing" - "time" -) - -func TestBuildIndexAndCandidates(t *testing.T) { - root := t.TempDir() - writeTestFile(t, root, "a.txt", "alpha needle omega\n") - writeTestFile(t, root, "b.txt", "nothing here\n") - indexPath := filepath.Join(t.TempDir(), "workspace.csearch") - - if err := BuildIndex(context.Background(), root, indexPath); err != nil { - t.Fatalf("BuildIndex() error = %v", err) - } - files, err := IndexCandidates(indexPath, "needle", false, false) - if err != nil { - t.Fatalf("IndexCandidates() error = %v", err) - } - if len(files) != 1 || files[0] != filepath.Join(root, "a.txt") { - t.Fatalf("candidates = %v", files) - } -} - -func TestWorkspaceFilesRespectsGitIgnore(t *testing.T) { - root := t.TempDir() - runGit(t, root, "init", "-q") - writeTestFile(t, root, ".gitignore", "ignored.txt\n") - writeTestFile(t, root, "tracked.txt", "tracked\n") - writeTestFile(t, root, "untracked.txt", "untracked\n") - writeTestFile(t, root, "ignored.txt", "ignored\n") - runGit(t, root, "add", ".gitignore", "tracked.txt") - - files, err := WorkspaceFiles(context.Background(), root) - if err != nil { - t.Fatalf("WorkspaceFiles() error = %v", err) - } - if slices.Contains(files, filepath.Join(root, "ignored.txt")) { - t.Fatalf("ignored file returned: %v", files) - } - for _, name := range []string{"tracked.txt", "untracked.txt"} { - if !slices.Contains(files, filepath.Join(root, name)) { - t.Fatalf("%s missing from %v", name, files) - } - } -} - -func TestWorkspaceFilesRejectsTrackedSymlinkOutsideRoot(t *testing.T) { - root := t.TempDir() - runGit(t, root, "init", "-q") - outside := filepath.Join(t.TempDir(), "secret.txt") - if err := os.WriteFile(outside, []byte("secret"), 0o600); err != nil { - t.Fatal(err) - } - link := filepath.Join(root, "link.txt") - if err := os.Symlink(outside, link); err != nil { - t.Fatal(err) - } - runGit(t, root, "add", "link.txt") - - files, err := WorkspaceFiles(context.Background(), root) - if err != nil { - t.Fatal(err) - } - _, accepted := confinedWorkspaceFile(root, link) - if slices.Contains(files, link) || accepted { - t.Fatalf("outside symlink accepted: %v", files) - } -} - -func TestConfinedWorkspaceFileRejectsIntermediateSymlinkEscape(t *testing.T) { - root := t.TempDir() - outside := t.TempDir() - writeTestFile(t, outside, "secret.txt", "secret") - if err := os.Symlink(outside, filepath.Join(root, "linkdir")); err != nil { - t.Fatal(err) - } - if path, ok := confinedWorkspaceFile(root, filepath.Join(root, "linkdir", "secret.txt")); ok { - t.Fatalf("intermediate symlink escape accepted as %s", path) - } -} - -func TestManagerRefreshBuildsAtomicallyAndRecordsGeneration(t *testing.T) { - root := t.TempDir() - dir := t.TempDir() - m := &Manager{ - root: root, indexPath: filepath.Join(dir, "workspace.csearch"), - dbPath: filepath.Join(dir, "indexes.db"), runner: writingRunner{}, - } - if err := m.ensureSchema(context.Background()); err != nil { - t.Fatal(err) - } - ctx := t.Context() - m.refreshAsync(ctx) - - // Poll for the async build to complete. Use a deadline longer than SQLite's - // busy timeout (5 s) so a contended CI runner doesn't trigger a false timeout. - deadline := time.Now().Add(8 * time.Second) - var installedPath string - for { - state, stateErr := m.state(context.Background()) - if stateErr == nil && !state.IndexedAt.IsZero() { - if _, err := os.Stat(state.IndexPath); err == nil { - installedPath = state.IndexPath - break - } - } - // If the goroutine errored, fail fast with the last error. - if stateErr == nil && state.IndexedAt.IsZero() { - // Check if status indicates an error. - db, dbErr := openIndexDB(m.dbPath) - if dbErr == nil { - var status, lastErr string - _ = db.QueryRowContext(context.Background(), - `SELECT status, last_error FROM workspace_indexes WHERE root = ?`, m.root).Scan(&status, &lastErr) - _ = db.Close() - if status == "error" { - t.Fatalf("build failed: %s", lastErr) - } - } - } - if time.Now().After(deadline) { - t.Fatal("index was not atomically installed") - } - time.Sleep(10 * time.Millisecond) - } - if installedPath == m.indexPath { - t.Fatalf("build did not install a generation-specific sidecar: %s", installedPath) - } - db, err := openIndexDB(m.dbPath) - if err != nil { - t.Fatal(err) - } - defer func() { _ = db.Close() }() - var status string - var generation int - if err := db.QueryRowContext(context.Background(), `SELECT status, generation FROM workspace_indexes WHERE root = ?`, root).Scan(&status, &generation); err != nil { - t.Fatal(err) - } - if status != "ready" || generation != 1 { - t.Fatalf("state = %q generation %d", status, generation) - } -} - -func TestStaleBuilderCannotCompleteNewerLease(t *testing.T) { - dir := t.TempDir() - m := &Manager{root: t.TempDir(), indexPath: filepath.Join(dir, "workspace.csearch"), dbPath: filepath.Join(dir, "indexes.db")} - if err := m.ensureSchema(context.Background()); err != nil { - t.Fatal(err) - } - oldStart := time.Now().UTC().Add(-2 * buildTimeout) - claimed, err := m.claimBuild(context.Background(), oldStart, "build-a") - if err != nil || !claimed { - t.Fatalf("first claim = %v, %v", claimed, err) - } - newStart := time.Now().UTC() - claimed, err = m.claimBuild(context.Background(), newStart, "build-b") - if err != nil || !claimed { - t.Fatalf("replacement claim = %v, %v", claimed, err) - } - if m.finishBuild(context.Background(), "build-a", "ready", oldStart, filepath.Join(dir, "a"), "") { - t.Fatal("stale builder completed replacement lease") - } - pathB := filepath.Join(dir, "b") - if !m.finishBuild(context.Background(), "build-b", "ready", newStart, pathB, "") { - t.Fatal("current builder could not complete lease") - } - state, err := m.state(context.Background()) - if err != nil || state.IndexPath != pathB { - t.Fatalf("state = %#v, %v", state, err) - } -} - -type writingRunner struct{} - -func (writingRunner) Build(_ context.Context, _, target string) error { - return os.WriteFile(target, []byte("index"), 0o600) -} - -func (writingRunner) Candidates(context.Context, string, string, bool, bool) ([]string, error) { - return nil, nil -} - -func writeTestFile(t *testing.T, dir, name, content string) { - t.Helper() - if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0o600); err != nil { - t.Fatal(err) - } -} - -func runGit(t *testing.T, dir string, args ...string) { - t.Helper() - cmd := exec.CommandContext(context.Background(), "git", append([]string{"-C", dir}, args...)...) - if out, err := cmd.CombinedOutput(); err != nil { - t.Fatalf("git %v: %v: %s", args, err, out) - } -} - -// A Manager built as a struct literal has no logger. Logging on the build -// path must not panic, since that path runs in a background goroutine where -// a panic takes the whole daemon down. -func TestManagerLoggerIsNilSafe(t *testing.T) { - tests := []struct { - name string - m *Manager - }{ - {name: "zero manager", m: &Manager{}}, - {name: "nil manager", m: nil}, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - if got := tc.m.logger(); got == nil { - t.Fatal("logger() returned nil") - } - tc.m.logger().Warn("must not panic") - }) - } -} diff --git a/internal/infrastructure/configuration/defaults.go b/internal/infrastructure/configuration/defaults.go index 120471b5b..a328625ba 100644 --- a/internal/infrastructure/configuration/defaults.go +++ b/internal/infrastructure/configuration/defaults.go @@ -231,7 +231,6 @@ func (l *Loader) applyGeneralDefaults(cfg *config.Config) { if cfg.DBPath == "" { cfg.DBPath = filepath.Join(l.dataHome(), "archie", "archie.db") } - cfg.Indexing = cfg.Indexing.WithDefaults(cfg.WorkDir) if cfg.PollInterval == 0 { cfg.PollInterval = config.Duration(defaultPollInterval) } diff --git a/internal/infrastructure/configuration/loader_test.go b/internal/infrastructure/configuration/loader_test.go index f61c14da3..07b1aca2a 100644 --- a/internal/infrastructure/configuration/loader_test.go +++ b/internal/infrastructure/configuration/loader_test.go @@ -904,55 +904,6 @@ func TestLoadBytesDoesNotExist(t *testing.T) { } } -func TestFinalizeIndexingDefaults(t *testing.T) { - tests := []struct { - name string - extra string - wantIndex string - wantDBIsSet bool - }{ - { - name: "defaults derive from work_dir", - // Both paths must land under the daemon's own work_dir so two - // archied instances on one host cannot share an index. - extra: "\nwork_dir = \"/srv/archie/work\"\n", - wantIndex: filepath.Join("/srv/archie/work", "indexes"), - wantDBIsSet: true, - }, - { - name: "explicit index_dir is preserved", - extra: "\nwork_dir = \"/srv/archie/work\"\n" + - "[indexing]\nindex_dir = \"/mnt/fast/idx\"\n", - wantIndex: "/mnt/fast/idx", - wantDBIsSet: true, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - path := filepath.Join(t.TempDir(), "config.toml") - // tc.extra precedes the table sections: a top-level key written - // after [[repos]] would be parsed as a member of that table. - contents := "bot_user = \"widget\"\n" + tc.extra + - "[forge]\ntype = \"gitea\"\nhost = \"https://git.example.test\"\ntoken_env = \"T\"\n" + - "[[repos]]\nowner = \"acme\"\nname = \"app\"\n" - if err := os.WriteFile(path, []byte(contents), 0o600); err != nil { - t.Fatal(err) - } - cfg, err := loadFile(path) - if err != nil { - t.Fatal(err) - } - if cfg.Indexing.IndexDir != tc.wantIndex { - t.Errorf("Indexing.IndexDir = %q, want %q", cfg.Indexing.IndexDir, tc.wantIndex) - } - if tc.wantDBIsSet && cfg.Indexing.DBPath == "" { - t.Error("Indexing.DBPath is empty; want a default derived from work_dir") - } - }) - } -} - func TestIdentitiesConfigParsesTwoIdentities(t *testing.T) { cfg, err := loadBytes([]byte(` bot_user = "default-legacy" diff --git a/internal/infrastructure/configuration/tomlwrite/tomlwrite_test.go b/internal/infrastructure/configuration/tomlwrite/tomlwrite_test.go index 98d55823d..4982f55d3 100644 --- a/internal/infrastructure/configuration/tomlwrite/tomlwrite_test.go +++ b/internal/infrastructure/configuration/tomlwrite/tomlwrite_test.go @@ -155,7 +155,7 @@ func TestApply_Idempotent_SecondRunLeavesOtherEditByteIdentical(t *testing.T) { } } -// [models], [indexing], [web], and [notify] in the real template all carry +// [models], [web], and [notify] in the real template all carry // a trailing "# ..." comment on the header line itself // ("[models] # role -> ..."). The header regexes were anchored with no // allowance for that, so Apply's table-tracking never recognised these as diff --git a/internal/releaseupdate/install_script_test.go b/internal/releaseupdate/install_script_test.go index 467d772c6..c84095519 100644 --- a/internal/releaseupdate/install_script_test.go +++ b/internal/releaseupdate/install_script_test.go @@ -904,7 +904,8 @@ func TestUpdateInstallBacksUpTheSiblingTaskDatabase(t *testing.T) { assertCallAbsent(t, calls, "--setenv=ARCHIE_TASK_DB_PATH="+configured+" ") } -// db_path also appears in other sections -- [indexing] owns its own database. +// db_path can also appear under a TOML table -- the retired [indexing] section +// once owned its own database there, and a future section could do the same. // Reading the first match in the file can select an unrelated database, so the // value must come from the daemon's own top-level key. func TestUpdateInstallIgnoresDbPathFromOtherSections(t *testing.T) { @@ -929,7 +930,7 @@ func TestUpdateInstallIgnoresDbPathFromOtherSections(t *testing.T) { assertCallContains(t, calls, "go run ./cmd/archie-state-store backup", "-db "+realStore) for _, call := range calls { if strings.Contains(call, "go run ./cmd/archie-state-store backup") && strings.Contains(call, indexStore) { - t.Errorf("backed up the indexing database %s: %q", indexStore, call) + t.Errorf("backed up the other section's database %s: %q", indexStore, call) } } assertCallAbsent(t, calls, "--setenv=ARCHIE_TASK_DB_PATH="+indexStore) diff --git a/internal/tools/builtin/builtin.go b/internal/tools/builtin/builtin.go index 7516c0d08..205e4835c 100644 --- a/internal/tools/builtin/builtin.go +++ b/internal/tools/builtin/builtin.go @@ -7,6 +7,8 @@ // - the docs tool is not registered and docs.go was not lifted. It serves // tau's own embedded documentation via the tau/docs package, which has // no meaning here. The pluginDocs parameter went with it. +// - the workspace codesearch index parameter was removed along with the +// index itself; RegisterBuiltins takes only the registry and cwd. // // Refresh by diffing against that path at a newer tau commit. Do not // edit without recording the change above. @@ -14,7 +16,7 @@ package builtin // RegisterBuiltins registers all built-in tools into the given registry. // The cwd parameter sets the working directory for file and shell operations. -func RegisterBuiltins(reg *Registry, cwd string, indexes ...GrepIndex) error { +func RegisterBuiltins(reg *Registry, cwd string) error { mq := NewMutationQueue() rt := NewReadTracker() @@ -23,7 +25,7 @@ func RegisterBuiltins(reg *Registry, cwd string, indexes ...GrepIndex) error { NewWriteTool(cwd, mq, rt), NewEditTool(cwd, mq, rt), NewShellTool(cwd, mq), - NewGrepTool(cwd, indexes...), + NewGrepTool(cwd), NewFindTool(cwd), } diff --git a/internal/tools/builtin/grep.go b/internal/tools/builtin/grep.go index f43477a5f..df78a29e3 100644 --- a/internal/tools/builtin/grep.go +++ b/internal/tools/builtin/grep.go @@ -14,6 +14,10 @@ // smaller helpers (resolveGrepTargets, runGrepBinary, grepExplicitTargets, // setGrepResultsRelPath, grepWalkVisitor) to satisfy golangci-lint // cyclop/gocognit limits. No behavioural change. +// - the workspace codesearch index integration was removed: GrepIndex, the +// candidate-narrowing path, the codesearch backend retry, and the +// search_backend metric are gone, and grep always walks the requested +// path directly. // // Refresh by diffing against that path at a newer tau commit. Do not // edit without recording the change above. @@ -67,12 +71,6 @@ type GrepParams struct { Limit int `json:"limit,omitempty"` // max matches to return } -// GrepIndex provides conservative workspace-wide candidate files. The grep -// tool remains responsible for authoritative matching and output formatting. -type GrepIndex interface { - Candidates(context.Context, string, bool, bool) ([]string, bool) -} - var grepSchema = Schema{ Name: "grep", Description: fmt.Sprintf("Search file contents for a regex pattern using ripgrep (rg). Respects .gitignore. Returns matching lines with file paths and line numbers. Supports alternation (e.g. 'foo|bar') and full regex syntax. Use context_before/context_after to show surrounding lines (capped at %d each). Output is capped at %d matches (adjustable via limit) and long lines are truncated to %d chars.", grepMaxContext, grepDefaultLimit, grepMaxLineChars), @@ -118,20 +116,16 @@ var grepSchema = Schema{ // NewGrepTool creates the built-in grep tool. -func NewGrepTool(cwd string, indexes ...GrepIndex) Tool { - var workspaceIndex GrepIndex - if len(indexes) > 0 { - workspaceIndex = indexes[0] - } +func NewGrepTool(cwd string) Tool { return Tool{ Schema: grepSchema, Source: "builtin", Emoji: "🔍", - Execute: makeGrepExecutor(cwd, workspaceIndex), + Execute: makeGrepExecutor(cwd), } } -func makeGrepExecutor(cwd string, workspaceIndex GrepIndex) Executor { +func makeGrepExecutor(cwd string) Executor { return func(ctx context.Context, params json.RawMessage, _ UIBridge) (Result, error) { var p GrepParams if err := json.Unmarshal(params, &p); err != nil { @@ -156,8 +150,6 @@ func makeGrepExecutor(cwd string, workspaceIndex GrepIndex) Executor { return Result{Content: "error: path escapes working directory", IsError: true, ErrorKind: "sandbox_escape"}, nil } - searchTargets, searchBackend := resolveGrepTargets(ctx, workspaceIndex, p, cwd, searchPath) - limit := p.Limit if limit <= 0 { limit = grepDefaultLimit @@ -165,55 +157,37 @@ func makeGrepExecutor(cwd string, workspaceIndex GrepIndex) Executor { binary, err := grepBinary() if err != nil { - searchBackend = "direct" // No external binary available - use pure-Go fallback. - output, err := grepFallback(ctx, p, searchPath, cwd, searchTargets) + output, err := grepFallback(ctx, p, searchPath, cwd) if err != nil { - return grepBackendResult(Result{Content: fmt.Sprintf("grep error: %v", err), IsError: true}, searchBackend, clamped), nil + return applyClampNotice(Result{Content: fmt.Sprintf("grep error: %v", err), IsError: true}, clamped), nil } if output == "" { - return grepBackendResult(Result{Content: "no matches found"}, searchBackend, clamped), nil + return applyClampNotice(Result{Content: "no matches found"}, clamped), nil } - return grepBackendResult(capGrepResult(output, limit), searchBackend, clamped), nil + return applyClampNotice(capGrepResult(output, limit), clamped), nil } - result, backend := runGrepBinary(ctx, binary, cwd, p, searchPath, searchTargets, searchBackend, limit) - return grepBackendResult(result, backend, clamped), nil + result := runGrepBinary(ctx, binary, cwd, p, searchPath, limit) + return applyClampNotice(result, clamped), nil } } -// resolveGrepTargets picks the search targets and backend label: the -// workspace index's narrowed candidates when it can serve the request, or a -// direct walk of searchPath otherwise. -// -// The index is workspace-wide, so it can serve any search scoped at or below -// cwd once its candidates are narrowed to the requested subtree. Requiring -// searchPath == cwd meant subdirectory searches silently fell back to a full -// walk, which is why the index served only 5% of searches across analysed -// sessions. -func resolveGrepTargets(ctx context.Context, workspaceIndex GrepIndex, p GrepParams, cwd, searchPath string) ([]string, string) { - if workspaceIndex != nil && isConfined(cwd, searchPath) { - if candidates, ok := workspaceIndex.Candidates(ctx, p.Pattern, p.Literal, p.CaseSensitive); ok { - if scoped := filterUnder(searchPath, candidates); len(scoped) > 0 { - return scoped, "codesearch" - } - // An empty scoped set is NOT evidence of no matches: the index - // may simply be cold or stale for this subtree, so fall through - // to the authoritative direct walk rather than reporting "no - // matches found" from an empty candidate list. - } +// applyClampNotice appends the context-clamp notice to a successful result. +func applyClampNotice(result Result, clampNotice string) Result { + if clampNotice != "" && !result.IsError { + result.Content += "\n" + clampNotice } - return []string{searchPath}, "direct" + return result } -// runGrepBinary runs the external rg binary against searchTargets, retrying -// against the authoritative searchPath if a codesearch-backed run fails, and -// returns a Result plus the backend that actually produced it. +// runGrepBinary runs the external rg binary against searchPath and returns +// the Result it produced. func runGrepBinary( ctx context.Context, binary, cwd string, p GrepParams, - searchPath string, searchTargets []string, searchBackend string, limit int, -) (Result, string) { - args := append(buildGrepArgs(p), searchTargets...) + searchPath string, limit int, +) Result { + args := append(buildGrepArgs(p), searchPath) cmd := exec.CommandContext(ctx, binary, args...) cmd.Dir = cwd @@ -222,22 +196,7 @@ func runGrepBinary( cmd.Stderr = &stderr err := cmd.Run() - output := stdout.String() - if searchBackend == "codesearch" && err != nil { - // Snapshot candidates can disappear, or an unusually large candidate - // argv can fail to spawn. Retry the authoritative workspace path. - searchBackend = "direct" - args = append(buildGrepArgs(p), searchPath) - cmd = exec.CommandContext(ctx, binary, args...) - cmd.Dir = cwd - stdout.Reset() - stderr.Reset() - cmd.Stdout = &stdout - cmd.Stderr = &stderr - err = cmd.Run() - output = stdout.String() - } if output == "" && err != nil { // Exit 1 means "no matches"; exit 2 means a real failure (bad // pattern, unreadable path). Reporting the latter as an empty @@ -246,31 +205,16 @@ func runGrepBinary( // error type. var exitErr *exec.ExitError if errors.As(err, &exitErr) && exitErr.ExitCode() == 1 { - return Result{Content: "no matches found"}, searchBackend + return Result{Content: "no matches found"} } errMsg := stderr.String() if errMsg == "" { errMsg = err.Error() } - return Result{Content: fmt.Sprintf("grep error: %s", errMsg), IsError: true}, searchBackend + return Result{Content: fmt.Sprintf("grep error: %s", errMsg), IsError: true} } - return capGrepResult(output, limit), searchBackend -} - -// filterUnder returns the candidates at or below root. Paths are compared -// after cleaning so that a candidate is only kept when root is a genuine path -// prefix, not merely a string prefix ("/a/bc" is not under "/a/b"). -func filterUnder(root string, candidates []string) []string { - root = filepath.Clean(root) - - out := make([]string, 0, len(candidates)) - for _, c := range candidates { - if isConfined(root, filepath.Clean(c)) { - out = append(out, c) - } - } - return out + return capGrepResult(output, limit) } // clampGrepContext bounds the context window in place and returns a notice @@ -294,14 +238,6 @@ func clampGrepContext(p *GrepParams) string { ) } -func grepBackendResult(result Result, backend, clampNotice string) Result { - result.MetricLabels = map[string]string{"search_backend": backend} - if clampNotice != "" && !result.IsError { - result.Content += "\n" + clampNotice - } - return result -} - func capGrepResult(output string, limit int) Result { lines := strings.Split(strings.TrimSuffix(output, "\n"), "\n") kept := make([]string, 0, len(lines)) @@ -349,23 +285,13 @@ func truncationBoundary(s string, limit int) int { } // grepFallback performs a pure-Go file scan for when ripgrep is not available. -// Works on all platforms including Windows. When targets has explicit file paths -// (not just the searchPath directory), only those files are searched. -func grepFallback(ctx context.Context, p GrepParams, searchPath, cwd string, targets []string) (string, error) { +// Works on all platforms including Windows. +func grepFallback(ctx context.Context, p GrepParams, searchPath, cwd string) (string, error) { matcher, err := buildMatcher(p) if err != nil { return "", err } - // When explicit files are provided (index candidates), search only those. - if len(targets) > 0 && targets[0] != searchPath { - results, err := grepExplicitTargets(ctx, targets, cwd, matcher, p) - if err != nil { - return "", err - } - return formatGrepResults(results), nil - } - var results []grepResult info, err := os.Stat(searchPath) if err != nil { @@ -386,26 +312,6 @@ func grepFallback(ctx context.Context, p GrepParams, searchPath, cwd string, tar return formatGrepResults(results), nil } -// grepExplicitTargets searches only the given files (index candidates), -// skipping any that fail to read. -func grepExplicitTargets(ctx context.Context, targets []string, cwd string, matcher func(string) bool, p GrepParams) ([]grepResult, error) { - var results []grepResult - for _, path := range targets { - select { - case <-ctx.Done(): - return nil, ctx.Err() - default: - } - res, fErr := grepFile(ctx, path, matcher, p.ContextBefore, p.ContextAfter) - if fErr != nil { - continue - } - setGrepResultsRelPath(res, cwd, path) - results = append(results, res...) - } - return results, nil -} - // setGrepResultsRelPath fills in each result's relPath relative to cwd, // falling back to the file's base name when the relative path collapses to // "" or ".". diff --git a/internal/tools/builtin/grep_test.go b/internal/tools/builtin/grep_test.go index bc763c98a..b1316776c 100644 --- a/internal/tools/builtin/grep_test.go +++ b/internal/tools/builtin/grep_test.go @@ -11,27 +11,6 @@ import ( "time" ) -type staticGrepIndex struct{ files []string } - -func (s staticGrepIndex) Candidates(context.Context, string, bool, bool) ([]string, bool) { - return s.files, true -} - -func TestGrepUsesIndexCandidatesAsAdvisoryFileSet(t *testing.T) { - tmp := t.TempDir() - indexed := filepath.Join(tmp, "indexed.txt") - createGrepTestFile(t, tmp, "indexed.txt", "needle\n") - createGrepTestFile(t, tmp, "excluded.txt", "needle\n") - tool := NewGrepTool(tmp, staticGrepIndex{files: []string{indexed}}) - result, err := tool.Execute(context.Background(), json.RawMessage(`{"pattern":"needle"}`), nil) - if err != nil || result.IsError { - t.Fatalf("grep result = %#v, err = %v", result, err) - } - if !strings.Contains(result.Content, "indexed.txt") || strings.Contains(result.Content, "excluded.txt") { - t.Fatalf("candidate-filtered output = %s", result.Content) - } -} - // TestGrepReportsSearchFailureRatherThanNoMatches guards the distinction // between ripgrep's exit 1 (no matches) and exit 2 (the search never ran). An // errorlint autofix once rewrote the check to match any *exec.ExitError and @@ -42,7 +21,7 @@ func TestGrepUsesIndexCandidatesAsAdvisoryFileSet(t *testing.T) { func TestGrepReportsSearchFailureRatherThanNoMatches(t *testing.T) { tmp := t.TempDir() createGrepTestFile(t, tmp, "haystack.txt", "needle\n") - tool := NewGrepTool(tmp, nil) + tool := NewGrepTool(tmp) result, err := tool.Execute(context.Background(), json.RawMessage(`{"pattern":"(unclosed"}`), nil) if err != nil { @@ -56,36 +35,6 @@ func TestGrepReportsSearchFailureRatherThanNoMatches(t *testing.T) { } } -func TestGrepRetriesDirectSearchWhenIndexedCandidatesAreStale(t *testing.T) { - tmp := t.TempDir() - createGrepTestFile(t, tmp, "live.txt", "needle\n") - missing := filepath.Join(tmp, "deleted.txt") - tool := NewGrepTool(tmp, staticGrepIndex{files: []string{missing}}) - result, err := tool.Execute(context.Background(), json.RawMessage(`{"pattern":"needle"}`), nil) - if err != nil || result.IsError { - t.Fatalf("grep result = %#v, err = %v", result, err) - } - if !strings.Contains(result.Content, "live.txt") || result.MetricLabels["search_backend"] != "direct" { - t.Fatalf("fallback output = %s, labels = %v", result.Content, result.MetricLabels) - } -} - -func TestGrepDiscardsPartialIndexedOutputOnCandidateError(t *testing.T) { - tmp := t.TempDir() - live := filepath.Join(tmp, "live.txt") - createGrepTestFile(t, tmp, "live.txt", "needle\n") - createGrepTestFile(t, tmp, "also-live.txt", "needle\n") - missing := filepath.Join(tmp, "deleted.txt") - tool := NewGrepTool(tmp, staticGrepIndex{files: []string{live, missing}}) - result, err := tool.Execute(context.Background(), json.RawMessage(`{"pattern":"needle"}`), nil) - if err != nil || result.IsError { - t.Fatalf("grep result = %#v, err = %v", result, err) - } - if !strings.Contains(result.Content, "also-live.txt") || result.MetricLabels["search_backend"] != "direct" { - t.Fatalf("partial indexed output was not replaced: %s, labels = %v", result.Content, result.MetricLabels) - } -} - func TestGrepFallback(t *testing.T) { // Create a temporary directory with test files. tmp := t.TempDir() @@ -196,7 +145,7 @@ func TestGrepFallback(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - got, err := grepFallback(ctx, tc.params, tc.params.Path, tmp, nil) + got, err := grepFallback(ctx, tc.params, tc.params.Path, tmp) if err != nil { t.Fatalf("grepFallback error: %v", err) } @@ -231,7 +180,7 @@ func TestGrepFallback_ContextDedup(t *testing.T) { defer cancel() params := GrepParams{Pattern: "Hello", Path: tmp, ContextBefore: 1, ContextAfter: 1} - got, err := grepFallback(ctx, params, tmp, tmp, nil) + got, err := grepFallback(ctx, params, tmp, tmp) if err != nil { t.Fatalf("grepFallback error: %v", err) } @@ -305,7 +254,7 @@ func TestGrepFallback_NonExistentPath(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - _, err := grepFallback(ctx, GrepParams{Pattern: "foo"}, "/nonexistent/path", "/nonexistent/path", nil) + _, err := grepFallback(ctx, GrepParams{Pattern: "foo"}, "/nonexistent/path", "/nonexistent/path") if err == nil { t.Fatal("expected error for non-existent path") } @@ -334,7 +283,7 @@ func TestGrepClampsContextLines(t *testing.T) { } createGrepTestFile(t, tmp, "f.txt", strings.Join(lines, "\n")+"\n") - tool := NewGrepTool(tmp, nil) + tool := NewGrepTool(tmp) res, err := tool.Execute(context.Background(), json.RawMessage( `{"pattern":"needle","context_before":40,"context_after":40}`, ), nil) @@ -362,7 +311,7 @@ func TestGrepLeavesModestContextAlone(t *testing.T) { tmp := t.TempDir() createGrepTestFile(t, tmp, "f.txt", "a\nb\nneedle\nd\ne\n") - tool := NewGrepTool(tmp, nil) + tool := NewGrepTool(tmp) res, err := tool.Execute(context.Background(), json.RawMessage( `{"pattern":"needle","context_before":2,"context_after":2}`, ), nil) @@ -378,60 +327,3 @@ func TestGrepLeavesModestContextAlone(t *testing.T) { } } } - -// The index served only 5% of searches across the analysed sessions -// (codesearch 11, direct 199) because it engaged only when the search path was -// exactly cwd. Any subdirectory-scoped search silently fell back to a walk. -func TestGrepUsesIndexForSubdirectorySearch(t *testing.T) { - tmp := t.TempDir() - createGrepTestFile(t, tmp, "sub/inside.txt", "needle\n") - createGrepTestFile(t, tmp, "other/outside.txt", "needle\n") - - index := staticGrepIndex{files: []string{ - filepath.Join(tmp, "sub", "inside.txt"), - filepath.Join(tmp, "other", "outside.txt"), - }} - - tool := NewGrepTool(tmp, index) - res, err := tool.Execute(context.Background(), json.RawMessage( - `{"pattern":"needle","path":"sub"}`, - ), nil) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if res.IsError { - t.Fatalf("unexpected tool error: %s", res.Content) - } - if got := res.MetricLabels["search_backend"]; got != "codesearch" { - t.Fatalf("search_backend = %q, want codesearch", got) - } - if !strings.Contains(res.Content, "inside.txt") { - t.Fatalf("expected the in-scope match, got:\n%s", res.Content) - } - if strings.Contains(res.Content, "outside.txt") { - t.Fatalf("candidates outside the search path must be filtered out, got:\n%s", res.Content) - } -} - -// If the index has no candidates inside the requested subtree we must not -// conclude "no matches" from an empty candidate list - fall back to a walk. -func TestGrepFallsBackToDirectWhenNoCandidatesInSubtree(t *testing.T) { - tmp := t.TempDir() - createGrepTestFile(t, tmp, "sub/inside.txt", "needle\n") - - index := staticGrepIndex{files: []string{filepath.Join(tmp, "elsewhere.txt")}} - - tool := NewGrepTool(tmp, index) - res, err := tool.Execute(context.Background(), json.RawMessage( - `{"pattern":"needle","path":"sub"}`, - ), nil) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if got := res.MetricLabels["search_backend"]; got != "direct" { - t.Fatalf("search_backend = %q, want direct", got) - } - if !strings.Contains(res.Content, "inside.txt") { - t.Fatalf("an empty candidate set must not hide real matches, got:\n%s", res.Content) - } -} diff --git a/internal/tools/builtin/pathutil_test.go b/internal/tools/builtin/pathutil_test.go index c0c22f0c0..a73b8579f 100644 --- a/internal/tools/builtin/pathutil_test.go +++ b/internal/tools/builtin/pathutil_test.go @@ -108,7 +108,7 @@ func TestGrepRejectsGoModCache(t *testing.T) { t.Fatal(err) } - tool := NewGrepTool(cwd, nil) + tool := NewGrepTool(cwd) res, err := tool.Execute(context.Background(), json.RawMessage( `{"pattern":"Needle","path":`+quote(cache)+`}`, ), nil) diff --git a/internal/tools/provider/builtin/provider.go b/internal/tools/provider/builtin/provider.go index fddce737d..74e731af0 100644 --- a/internal/tools/provider/builtin/provider.go +++ b/internal/tools/provider/builtin/provider.go @@ -32,11 +32,6 @@ type Provider struct { // config.ChatConfig.UnrestrictedFilesystem for when that is wanted. unrestricted bool - // workspaceIndex narrows grep's candidate files. Optional: nil leaves - // grep on its direct walk, which is the behaviour when no index is - // configured or its build has not finished. - workspaceIndex toolsbuiltin.GrepIndex - // registry holds the constructed tools. Built once in Start so // Discover cannot partially fail. registry *toolsbuiltin.Registry @@ -48,15 +43,8 @@ type Provider struct { // // unrestricted lets the file tools reach absolute paths outside workspace; // relative paths still resolve against it either way. -// A workspace index is optional and variadic to match -// toolsbuiltin.RegisterBuiltins, which has always taken one the same way. -// Only the first is used. -func New(workspace string, unrestricted bool, indexes ...toolsbuiltin.GrepIndex) *Provider { - p := &Provider{workspace: workspace, unrestricted: unrestricted} - if len(indexes) > 0 { - p.workspaceIndex = indexes[0] - } - return p +func New(workspace string, unrestricted bool) *Provider { + return &Provider{workspace: workspace, unrestricted: unrestricted} } // Manifest declares the adapter's tool capability. @@ -84,14 +72,7 @@ func (p *Provider) Start(ctx context.Context) error { toolsbuiltin.SetPathConfinement(!p.unrestricted) registry := toolsbuiltin.NewRegistry() - // Forwarded only when present. Passing a nil interface would still work - // today (grep nil-checks it), but an explicit branch keeps a typed-nil - // index from reaching a call on a nil receiver. - var indexes []toolsbuiltin.GrepIndex - if p.workspaceIndex != nil { - indexes = append(indexes, p.workspaceIndex) - } - if err := toolsbuiltin.RegisterBuiltins(registry, p.workspace, indexes...); err != nil { + if err := toolsbuiltin.RegisterBuiltins(registry, p.workspace); err != nil { return fmt.Errorf("workspace tool provider: register builtins: %w", err) } p.registry = registry diff --git a/internal/tools/provider/builtin/provider_test.go b/internal/tools/provider/builtin/provider_test.go index 61d75f5f9..fdd87df3e 100644 --- a/internal/tools/provider/builtin/provider_test.go +++ b/internal/tools/provider/builtin/provider_test.go @@ -5,7 +5,6 @@ import ( "os" "path/filepath" "strings" - "sync" "testing" provider "github.com/samcharles93/archie-core/internal/tools/provider/builtin" @@ -234,78 +233,6 @@ func TestUnrestrictedFilesystemReachesOutsideWorkspace(t *testing.T) { }) } -// stubGrepIndex records whether the grep tool consulted a supplied workspace -// index, and serves a fixed candidate set. -type stubGrepIndex struct { - mu sync.Mutex - calls int - pattern string - files []string -} - -func (s *stubGrepIndex) Candidates(_ context.Context, pattern string, _, _ bool) ([]string, bool) { - s.mu.Lock() - defer s.mu.Unlock() - s.calls++ - s.pattern = pattern - return s.files, true -} - -func (s *stubGrepIndex) observed() (int, string) { - s.mu.Lock() - defer s.mu.Unlock() - return s.calls, s.pattern -} - -// TestGrepConsultsSuppliedWorkspaceIndex covers the wiring this provider is -// the only production owner of. internal/tools/builtin.RegisterBuiltins has -// always accepted a GrepIndex, and nothing ever passed one, so every grep ran -// the direct walk regardless of configuration. A provider that accepts an -// index and then drops it on the floor is the exact failure worth a test: -// it looks wired from the outside and changes nothing. -func TestGrepConsultsSuppliedWorkspaceIndex(t *testing.T) { - t.Parallel() - - dir := t.TempDir() - if err := os.WriteFile(filepath.Join(dir, "hit.go"), []byte("package p\n\nfunc Target() {}\n"), 0o600); err != nil { - t.Fatal(err) - } - - idx := &stubGrepIndex{files: []string{filepath.Join(dir, "hit.go")}} - - p := provider.New(dir, false, idx) - if err := p.Start(context.Background()); err != nil { - t.Fatalf("Start: %v", err) - } - t.Cleanup(func() { _ = p.Stop(context.Background()) }) - - entries, err := p.Discover(context.Background()) - if err != nil { - t.Fatalf("Discover: %v", err) - } - var grep func(context.Context, map[string]any) (any, error) - for _, e := range entries { - if e.Name == "grep" { - grep = e.Handler - } - } - if grep == nil { - t.Fatal("no grep tool discovered") - } - - if _, err := grep(context.Background(), map[string]any{"pattern": "Target"}); err != nil { - t.Fatalf("grep handler: %v", err) - } - - calls, pattern := idx.observed() - if calls == 0 { - t.Fatal("grep never consulted the supplied workspace index, so the index is decoded, constructed and ignored") - } - if pattern != "Target" { - t.Errorf("index asked for pattern %q, want the caller's pattern %q", pattern, "Target") - } -} - // TestWorkspaceToolsCarryTheirOwnIcon pins that each builtin declares an icon // and that the conversion carries it. A chat surface renders what the tool // registered, so an icon missing here is an icon missing on screen. diff --git a/scripts/archie-update-install b/scripts/archie-update-install index 4f61c0f83..131b7cda2 100755 --- a/scripts/archie-update-install +++ b/scripts/archie-update-install @@ -50,8 +50,9 @@ configured_agent_image() { # rollback would then restore that empty file over the database and delete its # -wal/-shm -- destroying the task store it exists to protect. # -# Only the top-level key counts. db_path also appears in other sections -# ([indexing] owns its own database), and the first match in the file can +# Only the top-level key counts. db_path can also appear under a TOML table +# (the retired [indexing] section once owned its own database), and the first +# match in the file can # easily be one of those. configured_task_db_path() { local base From 97f55793c41cc7bf0fe0b5141f7de4b6dc7f2a94 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Wed, 23 Sep 2026 00:37:15 +1000 Subject: [PATCH 03/20] feat(stateadmin): read-only operator dashboard over the State Store Serve an optional PocketBase surface over archie.db from the process that already owns it, behind archie-state-store -admin-addr. It co-tenants on -tasks.sqlite via PocketBase's DBConnect hook rather than opening a database of its own, so the single-owner ownership claim covers it and PRAGMA user_version stays archie's. Every archie table is exposed as a view collection, which is read-only by construction. Views alone do not hold that line: PocketBase also ships POST /api/sql, which executes write statements as well as reads, and /api/backups, which would put a second snapshot and restore semantics beside store.Backup's VACUUM INTO. Both routes are refused rather than left behind a superuser check, because a superuser writing task state is still a second writer. The gRPC contract remains the only writer. apis.NewRouter binds the JSON API but not the dashboard (apis.Serve mounts the embedded UI itself), so /_/ is mounted here and the caller keeps the listener. The dependency pin and the Taskfile/README changes for this landed with the preceding commit, which swept up the staged files. --- cmd/archie-state-store/main.go | 1 + internal/app/archied/state_store.go | 60 +++++- .../infrastructure/stateadmin/stateadmin.go | 181 ++++++++++++++++++ .../stateadmin/stateadmin_test.go | 168 ++++++++++++++++ 4 files changed, 406 insertions(+), 4 deletions(-) create mode 100644 internal/infrastructure/stateadmin/stateadmin.go create mode 100644 internal/infrastructure/stateadmin/stateadmin_test.go diff --git a/cmd/archie-state-store/main.go b/cmd/archie-state-store/main.go index c664fa746..f9da1e4e8 100644 --- a/cmd/archie-state-store/main.go +++ b/cmd/archie-state-store/main.go @@ -39,6 +39,7 @@ func run() int { flag.StringVar(&options.Listen, "listen", "", "state store gRPC listen address (defaults to [services.state].listen, else 127.0.0.1:9090)") flag.StringVar(&options.Token, "token", "", "bearer token required for a non-loopback listener (defaults to [services.state].target_token)") flag.StringVar(&options.ReadyAddr, "ready-addr", "", "optional readiness HTTP listen address (e.g. 127.0.0.1:9091)") + flag.StringVar(&options.AdminAddr, "admin-addr", "", "optional read-only operator dashboard listen address (e.g. 127.0.0.1:8090)") flag.Usage = func() { fmt.Fprintf(flag.CommandLine.Output(), "usage: archie-state-store [flags] # serve the State Store gRPC contract\n\noffline recovery: archie-state-store -h\n\n") flag.PrintDefaults() diff --git a/internal/app/archied/state_store.go b/internal/app/archied/state_store.go index 6c3779b97..1fe82d3ef 100644 --- a/internal/app/archied/state_store.go +++ b/internal/app/archied/state_store.go @@ -30,6 +30,7 @@ import ( "github.com/samcharles93/archie-core/internal/domain/storecontract" "github.com/samcharles93/archie-core/internal/infrastructure/cronstore" "github.com/samcharles93/archie-core/internal/infrastructure/readiness" + "github.com/samcharles93/archie-core/internal/infrastructure/stateadmin" "github.com/samcharles93/archie-core/internal/infrastructure/staterpc" "github.com/samcharles93/archie-core/internal/store" ) @@ -65,6 +66,10 @@ type StateStoreOptions struct { // address: GET /healthz (liveness) and GET /health/detailed (the // state_db probe). Empty disables it. ReadyAddr string + // AdminAddr, when non-empty, starts the read-only operator dashboard on + // the address. It co-tenants on the same SQLite file this process already + // owns, so it needs no lock and no database of its own. Empty disables it. + AdminAddr string } // RunStateStore owns the single archie.db SQLite file, registers every @@ -131,10 +136,8 @@ func RunStateStore(ctx context.Context, options StateStoreOptions) error { defer listener.Close() b.log.Info("archie-state-store running", "addr", listener.Addr().String(), "token_required", !loopback) - if options.ReadyAddr != "" { - if err := b.startStateStoreReadiness(ctx, options.ReadyAddr); err != nil { - return err - } + if err := b.startOptionalSurfaces(ctx, options); err != nil { + return err } deps := b.stateStoreDeps(grants) @@ -421,6 +424,55 @@ func (b *boot) startStateStoreReadiness(ctx context.Context, readyAddr string) e return b.serveHealth(ctx, readyAddr, mux, "state store readiness") } +// startOptionalSurfaces starts the HTTP surfaces this process serves beside +// the gRPC contract, each enabled only by its own flag. +// +// They are gathered here rather than branched inline because RunStateStore is +// at its complexity budget: a surface that is off by default belongs to its +// own decision, not to the boot sequence that triggers it -- the same reason +// reportUnseededResources is a method. +func (b *boot) startOptionalSurfaces(ctx context.Context, options StateStoreOptions) error { + if options.ReadyAddr != "" { + if err := b.startStateStoreReadiness(ctx, options.ReadyAddr); err != nil { + return err + } + } + if options.AdminAddr != "" { + if err := b.startStateAdmin(ctx, options.AdminAddr); err != nil { + return err + } + } + return nil +} + +// startStateAdmin starts the read-only operator dashboard on adminAddr. +// +// It is started here, inside the process that already holds the database's +// ownership claim (openStateStore above), because that claim is what makes a +// second reader safe to co-tenant: a separate admin process would be a second +// opener of a single-owner file. The surface exposes view collections only and +// refuses the routes that could write, so the StateStoreService contract this +// process serves stays the only writer of task state. +func (b *boot) startStateAdmin(ctx context.Context, adminAddr string) error { + admin, err := stateadmin.New(stateadmin.Config{ + DBPath: taskDBPath(b.cfg.DBPath), + DataDir: filepath.Join(filepath.Dir(taskDBPath(b.cfg.DBPath)), "admin"), + }) + if err != nil { + return err + } + b.addCleanup(func() { + if err := admin.Close(); err != nil { + b.log.Error("close state admin", "err", err) + } + }) + handler, err := admin.Handler() + if err != nil { + return fmt.Errorf("build state admin handler: %w", err) + } + return b.serveHealth(ctx, adminAddr, handler, "state store admin") +} + // stateStoreServerOpts applies the transport security boundary (§9): a // loopback listener is served insecure with no token; any non-loopback // address requires a bearer token, else the process fails closed. It cannot diff --git a/internal/infrastructure/stateadmin/stateadmin.go b/internal/infrastructure/stateadmin/stateadmin.go new file mode 100644 index 000000000..8ae03a45a --- /dev/null +++ b/internal/infrastructure/stateadmin/stateadmin.go @@ -0,0 +1,181 @@ +// Package stateadmin serves a read-only operator surface over the State +// Store's SQLite file: a dashboard for inspecting task state, backed by +// PocketBase. +// +// It co-tenants on archie.db rather than owning a database of its own. +// PocketBase's data.db is redirected onto the State Store's file, so its +// system tables sit beside the State Store's in one file, and PRAGMA +// user_version -- archie's migration version -- is left untouched. Running +// inside the State Store process puts the surface under the same +// store.AcquireOwnership claim, so the single-owner invariant covers it +// without a second lock. +// +// Every archie table is exposed as a PocketBase *view* collection, which is +// read-only by construction. That is the point: the StateStoreService gRPC +// contract stays the only writer of task state, and this surface cannot +// become a second one. +// +// View collections alone are not enough to hold that line. PocketBase also +// ships two superuser routes that reach past them, so both are refused here: +// POST /api/sql executes write statements as well as reads (apis.executeQuery +// detects a write and runs it anyway), and /api/backups would give the file a +// second snapshot and restore semantics next to store.Backup's VACUUM INTO. +// Refusing the routes removes them rather than leaving them behind an auth +// check, because "only a superuser can corrupt task state" is still a second +// writer. +package stateadmin + +import ( + "errors" + "fmt" + "net/http" + "path/filepath" + "strings" + + "github.com/pocketbase/dbx" + "github.com/pocketbase/pocketbase" + "github.com/pocketbase/pocketbase/apis" + "github.com/pocketbase/pocketbase/core" + "github.com/pocketbase/pocketbase/tools/router" + "github.com/pocketbase/pocketbase/ui" + _ "modernc.org/sqlite" +) + +// refusedPrefixes are the PocketBase routes that would make this surface a +// second writer of State Store data. See the package comment. +var refusedPrefixes = []string{"/api/backups", "/api/sql"} + +// views maps each exposed collection to the query that populates it. The +// collection name is prefixed so it cannot collide with a PocketBase system +// collection, and every query must select an "id": PocketBase keys view +// records on it, and resources is keyed by kind rather than a row id. +var views = []struct { + name string + query string +}{ + {"archie_tasks", `SELECT id, owner, repo, issue_number, title, status, workflow, stage, branch, pr_number, tokens_used, iterations, attempt, park_reason, identity, created_at, updated_at FROM tasks`}, + {"archie_transitions", `SELECT id, task_id, at, from_status, to_status, detail FROM transitions`}, + {"archie_events", `SELECT id, at, kind, task_id, repo, issue, workflow, stage, attempt FROM events`}, + {"archie_bindings", `SELECT id, name, source, mapping_id, workflow, owner, repo, version, status FROM bindings`}, + {"archie_identities", `SELECT id, kind, display_name, lifecycle, version, created_at, updated_at FROM identities`}, + {"archie_resources", `SELECT kind AS id, kind, version, updated_at FROM resources`}, +} + +// ViewCollections returns the collection names this surface exposes. +func ViewCollections() []string { + names := make([]string, 0, len(views)) + for _, v := range views { + names = append(names, v.name) + } + return names +} + +// Config describes one admin surface. +type Config struct { + // DBPath is the State Store's SQLite file. The surface co-tenants on it. + DBPath string + // DataDir holds PocketBase's own auxiliary.db, settings and uploads. + // Nothing archie owns lives here. + DataDir string +} + +// Server is a bootstrapped admin surface. +type Server struct { + app *pocketbase.PocketBase +} + +// New bootstraps the surface against cfg.DBPath and registers its view +// collections. It binds no listener: the caller serves Handler on its own, +// so this package has no lifecycle of its own to get wrong. +func New(cfg Config) (*Server, error) { + if cfg.DBPath == "" { + return nil, errors.New("stateadmin: DBPath is required") + } + if cfg.DataDir == "" { + return nil, errors.New("stateadmin: DataDir is required") + } + app := pocketbase.NewWithConfig(pocketbase.Config{ + DefaultDataDir: cfg.DataDir, + HideStartBanner: true, + DBConnect: connectTo(cfg.DBPath), + }) + if err := app.Bootstrap(); err != nil { + return nil, fmt.Errorf("stateadmin: bootstrap: %w", err) + } + // apis.Serve would run these on listen; this surface is served through the + // caller's listener instead, so it applies them itself. + if err := app.RunAllMigrations(); err != nil { + return nil, fmt.Errorf("stateadmin: run migrations: %w", err) + } + if err := registerViews(app); err != nil { + return nil, err + } + return &Server{app: app}, nil +} + +// connectTo redirects PocketBase's data.db onto the State Store's file while +// leaving auxiliary.db where PocketBase wants it. The DSN matches the one the +// store opens with, so both connections agree on journal mode and busy +// timeout instead of racing on conflicting pragmas. +func connectTo(dbPath string) core.DBConnectFunc { + return func(requested string) (*dbx.DB, error) { + if filepath.Base(requested) == "data.db" { + requested = dbPath + } + return dbx.Open("sqlite", requested+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)") + } +} + +// registerViews creates or updates each view collection. It runs on every +// process start, so an existing collection is updated rather than refused. +func registerViews(app core.App) error { + for _, v := range views { + collection, err := app.FindCollectionByNameOrId(v.name) + if err != nil { + collection = core.NewViewCollection(v.name) + } else if !collection.IsView() { + return fmt.Errorf("stateadmin: collection %q exists and is not a view", v.name) + } + collection.ViewQuery = v.query + if err := app.Save(collection); err != nil { + return fmt.Errorf("stateadmin: register view %q: %w", v.name, err) + } + } + return nil +} + +// refuseWritePaths answers every refusedPrefixes route with 404. +func refuseWritePaths(r *router.Router[*core.RequestEvent]) { + r.BindFunc(func(e *core.RequestEvent) error { + for _, prefix := range refusedPrefixes { + if strings.HasPrefix(e.Request.URL.Path, prefix) { + return router.NewNotFoundError("", nil) + } + } + return e.Next() + }) +} + +// Handler builds the surface's HTTP handler without binding a listener. +// +// apis.NewRouter binds the JSON API but not the dashboard: apis.Serve mounts +// the embedded UI on /_/ itself, along with autocert and www-redirect +// machinery this surface has no use for. So the dashboard route is mounted +// here, and the caller keeps ownership of the listener. +func (s *Server) Handler() (http.Handler, error) { + r, err := apis.NewRouter(s.app) + if err != nil { + return nil, err + } + refuseWritePaths(r) + if ui.DistDirFS != nil { + r.GET("/_/{path...}", apis.Static(ui.DistDirFS, false)).Bind(apis.Gzip()) + } + return r.BuildMux() +} + +// App exposes the underlying PocketBase application. +func (s *Server) App() core.App { return s.app } + +// Close releases the surface's database connections. +func (s *Server) Close() error { return s.app.ClearBootstrap() } diff --git a/internal/infrastructure/stateadmin/stateadmin_test.go b/internal/infrastructure/stateadmin/stateadmin_test.go new file mode 100644 index 000000000..03a6b3fbf --- /dev/null +++ b/internal/infrastructure/stateadmin/stateadmin_test.go @@ -0,0 +1,168 @@ +package stateadmin_test + +import ( + "errors" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + + "github.com/samcharles93/archie-core/internal/domain/workflow" + "github.com/samcharles93/archie-core/internal/infrastructure/stateadmin" + "github.com/samcharles93/archie-core/internal/store" +) + +// openArchieDB creates a real, migrated State Store database and returns its +// path. The admin surface co-tenants on this exact file, so the tests exercise +// the production schema rather than a stand-in. +func openArchieDB(t *testing.T) string { + t.Helper() + path := filepath.Join(t.TempDir(), "archie.db-tasks.sqlite") + st, err := store.Open(t.Context(), path) + if err != nil { + t.Fatalf("store.Open() error = %v", err) + } + t.Cleanup(func() { _ = st.Close() }) + return path +} + +func newAdmin(t *testing.T, dbPath string) *stateadmin.Server { + t.Helper() + srv, err := stateadmin.New(stateadmin.Config{ + DBPath: dbPath, + DataDir: filepath.Join(t.TempDir(), "pb_data"), + }) + if err != nil { + t.Fatalf("stateadmin.New() error = %v", err) + } + t.Cleanup(func() { _ = srv.Close() }) + return srv +} + +// TestViewCollectionsExposeStateTables pins that every table the admin surface +// claims to expose is registered as a *view* collection. View collections are +// read-only by construction, which is what keeps the gRPC contract the only +// writer of task state. +func TestViewCollectionsExposeStateTables(t *testing.T) { + srv := newAdmin(t, openArchieDB(t)) + + for _, name := range stateadmin.ViewCollections() { + collection, err := srv.App().FindCollectionByNameOrId(name) + if err != nil { + t.Fatalf("FindCollectionByNameOrId(%q) error = %v, want the collection to exist", name, err) + } + if !collection.IsView() { + t.Errorf("collection %q type = %q, want a view: a base collection would let the admin UI write task state behind the gRPC contract", name, collection.Type) + } + } +} + +// TestBootstrapPreservesSchemaVersion pins that PocketBase's own bootstrap does +// not disturb archie's migration versioning. Both systems share one file; +// PRAGMA user_version is archie's and must survive co-tenancy untouched. +func TestBootstrapPreservesSchemaVersion(t *testing.T) { + path := openArchieDB(t) + + before, err := store.ValidateFile(t.Context(), path) + if err != nil { + t.Fatalf("ValidateFile() before admin bootstrap error = %v", err) + } + + newAdmin(t, path) + + after, err := store.ValidateFile(t.Context(), path) + if err != nil { + t.Fatalf("ValidateFile() after admin bootstrap error = %v, want the store to still validate", err) + } + if before != after { + t.Errorf("schema version moved across admin bootstrap: before = %d, after = %d", before, after) + } +} + +// TestStoreWritesSurviveCoTenancy is the regression that matters: archie's +// conditional-UPDATE optimistic concurrency must keep working, and keep +// reporting ErrStaleTransition for a losing writer, while the admin surface +// holds its own connections to the same file. +func TestStoreWritesSurviveCoTenancy(t *testing.T) { + path := openArchieDB(t) + newAdmin(t, path) + + st, err := store.Open(t.Context(), path) + if err != nil { + t.Fatalf("store.Open() alongside admin error = %v", err) + } + defer st.Close() + + if _, err := st.EnqueueIssue(t.Context(), "o", "r", 1, "co-tenancy", "", "", ""); err != nil { + t.Fatalf("EnqueueIssue() while admin holds the file error = %v", err) + } + queued, err := st.TaskByIssue(t.Context(), "o", "r", 1) + if err != nil { + t.Fatalf("TaskByIssue() error = %v", err) + } + id := queued.ID + + if err := st.Transition(t.Context(), id, workflow.StatusQueued, workflow.StatusRunning, ""); err != nil { + t.Fatalf("Transition(queued->running) error = %v, want the write to commit under co-tenancy", err) + } + // The same transition again must lose: the row is no longer queued. + if err := st.Transition(t.Context(), id, workflow.StatusQueued, workflow.StatusRunning, ""); !errors.Is(err, store.ErrStaleTransition) { + t.Fatalf("second Transition() error = %v, want ErrStaleTransition: optimistic concurrency must not degrade under co-tenancy", err) + } +} + +// TestWriteReachingRoutesAreRefused pins that the routes which reach past the +// read-only view collections are gone, not merely authenticated. POST /api/sql +// executes write statements (apis.executeQuery runs a detected write anyway), +// and /api/backups would add a second snapshot and restore path beside +// store.Backup. Both are superuser-gated upstream, which is not the same as +// absent: a superuser writing task state is still a second writer. +func TestWriteReachingRoutesAreRefused(t *testing.T) { + srv := newAdmin(t, openArchieDB(t)) + + mux, err := srv.Handler() + if err != nil { + t.Fatalf("Handler() error = %v", err) + } + // The method must match the route PocketBase actually binds: a GET against + // the POST-only /api/sql would 404 on method alone and assert nothing. + for _, route := range []struct{ method, path string }{ + {http.MethodGet, "/api/backups"}, + {http.MethodGet, "/api/backups/some-snapshot"}, + {http.MethodPost, "/api/sql"}, + } { + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, httptest.NewRequestWithContext(t.Context(), route.method, route.path, nil)) + if rec.Code != http.StatusNotFound { + t.Errorf("%s %s status = %d, want 404: the gRPC contract is the only writer and store.Backup the only snapshot path", route.method, route.path, rec.Code) + } + } +} + +// TestViewCollectionsAreIdempotent pins that a restart re-registers cleanly. +// The admin surface bootstraps on every process start, so a second pass over an +// already-populated database must update rather than fail. +func TestViewCollectionsAreIdempotent(t *testing.T) { + path := openArchieDB(t) + newAdmin(t, path) + newAdmin(t, path) +} + +// TestDashboardIsServed pins that the operator dashboard is actually reachable. +// apis.NewRouter binds the JSON API only; the embedded UI is mounted by +// apis.Serve, which this surface does not use. Without an explicit mount /_/ +// answers 404 and the surface has an API but no dashboard, which is the whole +// reason it exists. +func TestDashboardIsServed(t *testing.T) { + srv := newAdmin(t, openArchieDB(t)) + + mux, err := srv.Handler() + if err != nil { + t.Fatalf("Handler() error = %v", err) + } + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/_/", nil)) + if rec.Code != http.StatusOK { + t.Errorf("GET /_/ status = %d, want 200: the dashboard is the surface's reason to exist", rec.Code) + } +} From 3ccc57d445b4b9fe142ebfcb4299812e03faab0f Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Wed, 23 Sep 2026 00:55:57 +1000 Subject: [PATCH 04/20] fix(stateadmin): drop the default users collection and pin the sqlite driver PocketBase creates a "users" auth collection on first bootstrap whose createRule is open, so an unauthenticated caller could POST a record and write a row into the State Store's database file. This surface authenticates superusers only and has no notion of end users, so the collection is deleted rather than given a tighter rule: one that is absent cannot be loosened again by a later PocketBase default. Also re-applies the modernc pin. It was applied while developing the previous commit but a concurrent go.mod resolution reverted it before that commit was taken, so the tree has been running 1.59.0/1.77.1 against a PocketBase built and tested on 1.57.0/1.74.4, printing the mismatch warning on every start. Note for future dependency work: go get -u will undo this pin, and modernc.org/libc does not follow semver, which is why PocketBase checks. The claim in the previous commit message that the pin landed there is wrong. --- go.mod | 4 +-- go.sum | 7 +++++ .../infrastructure/stateadmin/stateadmin.go | 22 +++++++++++++++ .../stateadmin/stateadmin_test.go | 28 +++++++++++++++++++ 4 files changed, 59 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index a5f4658a4..6b6443806 100644 --- a/go.mod +++ b/go.mod @@ -71,7 +71,7 @@ require ( golang.org/x/time v0.16.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260921155816-b14227669459 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260921155816-b14227669459 // indirect - modernc.org/libc v1.77.1 // indirect + modernc.org/libc v1.74.4 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.12.1 // indirect ) @@ -101,5 +101,5 @@ require ( google.golang.org/grpc v1.85.0-dev.0.20260825072537-93e31b48545e google.golang.org/protobuf v1.36.12 gopkg.in/yaml.v3 v3.0.1 - modernc.org/sqlite v1.59.0 + modernc.org/sqlite v1.57.0 ) diff --git a/go.sum b/go.sum index deabe5645..b4c2313cc 100644 --- a/go.sum +++ b/go.sum @@ -298,8 +298,10 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= +modernc.org/cc/v4 v4.29.1 h1:MKgdCV3WykTSPqpVrnxdEDS0HEd2FHpKZDzxzU5LyeI= modernc.org/cc/v4 v4.29.7 h1:q+NXGJ0bK3b4TXFYQQVr9pYETGnmwFWkrUzJnMya/Tg= modernc.org/cc/v4 v4.29.7/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU= modernc.org/ccgo/v4 v4.36.0 h1:T5h7ZMcrw2r94oDLtBcU3YI6e1W8DiUg2SJIs2fwD2s= modernc.org/ccgo/v4 v4.36.0/go.mod h1:rrtGc2QkS239nYb/mQNuBMyjq3/y3ZXWbBjPoV3wqzA= modernc.org/ccgo/v4 v4.36.1 h1:ZNIUZAryN0UgnJwtyxrdEzcFc3yD4Cu4AzjfPXsLsIE= @@ -307,10 +309,13 @@ modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI= modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc= modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.74.4 h1:fX1Omw4o2/1C2iRkkIsrQTasJQldLhRmuPreXLoWs9k= +modernc.org/libc v1.74.4/go.mod h1:eeQAS9W3sZeKYMFubydxJpII9ybHWshk+7or7bLG9co= modernc.org/libc v1.77.0 h1:wV+faa9ixRoUNlakU0fHLi/1kV2YLG8D8Ksk3c2mPtY= modernc.org/libc v1.77.0/go.mod h1:BmXDCJcMkRMRbX2EI8+Zhh4fKWlHNbCCZEy0tiZevP4= modernc.org/libc v1.77.1 h1:Ct8j47QtiZ1Enj2DtFXQtUqrPCAjdCmPjtCuvrYQ0Hs= @@ -323,6 +328,8 @@ modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.57.0 h1:qNQP6xnx5M0ISNtlnxoOX0+cD5bJ0/gr9aMmndFczzg= +modernc.org/sqlite v1.57.0/go.mod h1:yCJ2cmAaIkHQ25oXWrF8H4O1lIfPYPR26yCEDj2P3pQ= modernc.org/sqlite v1.59.0 h1:X1es1GpqBlS/5T+vbM4HLUdaa8OtQx468DF2vrx+38A= modernc.org/sqlite v1.59.0/go.mod h1:+paeT2A3iPRHkQDwG7oA6Tk0zQd5woMEI8q7orfry8k= modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= diff --git a/internal/infrastructure/stateadmin/stateadmin.go b/internal/infrastructure/stateadmin/stateadmin.go index 8ae03a45a..05a378037 100644 --- a/internal/infrastructure/stateadmin/stateadmin.go +++ b/internal/infrastructure/stateadmin/stateadmin.go @@ -107,6 +107,9 @@ func New(cfg Config) (*Server, error) { if err := app.RunAllMigrations(); err != nil { return nil, fmt.Errorf("stateadmin: run migrations: %w", err) } + if err := removeDefaultAuthCollection(app); err != nil { + return nil, err + } if err := registerViews(app); err != nil { return nil, err } @@ -144,6 +147,25 @@ func registerViews(app core.App) error { return nil } +// removeDefaultAuthCollection deletes the "users" collection PocketBase +// creates on first bootstrap. +// +// Its createRule is open, so leaving it in place lets an unauthenticated +// caller POST a record and write a row into the State Store's database file. +// This surface authenticates superusers only and has no notion of end users, +// so the collection is deleted rather than given a tighter rule: a collection +// that is not there cannot be loosened again by a later PocketBase default. +func removeDefaultAuthCollection(app core.App) error { + collection, err := app.FindCollectionByNameOrId("users") + if err != nil { + return nil // already absent, including on every restart after the first + } + if err := app.Delete(collection); err != nil { + return fmt.Errorf("stateadmin: remove default users collection: %w", err) + } + return nil +} + // refuseWritePaths answers every refusedPrefixes route with 404. func refuseWritePaths(r *router.Router[*core.RequestEvent]) { r.BindFunc(func(e *core.RequestEvent) error { diff --git a/internal/infrastructure/stateadmin/stateadmin_test.go b/internal/infrastructure/stateadmin/stateadmin_test.go index 03a6b3fbf..b867bae7f 100644 --- a/internal/infrastructure/stateadmin/stateadmin_test.go +++ b/internal/infrastructure/stateadmin/stateadmin_test.go @@ -5,6 +5,7 @@ import ( "net/http" "net/http/httptest" "path/filepath" + "strings" "testing" "github.com/samcharles93/archie-core/internal/domain/workflow" @@ -166,3 +167,30 @@ func TestDashboardIsServed(t *testing.T) { t.Errorf("GET /_/ status = %d, want 200: the dashboard is the surface's reason to exist", rec.Code) } } + +// TestNoOpenRegistration pins that the surface has no self-service signup. +// PocketBase creates a "users" auth collection on first bootstrap whose +// createRule is open, so an unauthenticated caller could POST a record and +// write a row into the State Store's own database file. The surface is +// superuser-only by design, so the collection is removed rather than left +// with a tightened rule: a collection that does not exist cannot regress. +func TestNoOpenRegistration(t *testing.T) { + srv := newAdmin(t, openArchieDB(t)) + + if _, err := srv.App().FindCollectionByNameOrId("users"); err == nil { + t.Error("the default users collection still exists; it accepts unauthenticated record creation into archie's database file") + } + + mux, err := srv.Handler() + if err != nil { + t.Fatalf("Handler() error = %v", err) + } + body := strings.NewReader(`{"email":"nobody@example.com","password":"hunter2hunter2","passwordConfirm":"hunter2hunter2"}`) + req := httptest.NewRequestWithContext(t.Context(), http.MethodPost, "/api/collections/users/records", body) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, req) + if rec.Code < 400 { + t.Errorf("POST /api/collections/users/records status = %d, want a refusal: unauthenticated writes must not reach the State Store's file", rec.Code) + } +} From ad52528e048f226cd3e8dfd0ef431b673adaaef9 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Wed, 23 Sep 2026 00:56:24 +1000 Subject: [PATCH 05/20] fix(stateadmin): distinguish a missing collection from a failed lookup Both collection lookups treated any error as "not present". For the users collection that returned nil on a real failure (caught by nilerr); for the views it was worse but unflagged, because creating a fresh view collection on the strength of a transient lookup error is a schema change made for the wrong reason. Only sql.ErrNoRows now means absent. --- .../infrastructure/stateadmin/stateadmin.go | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/internal/infrastructure/stateadmin/stateadmin.go b/internal/infrastructure/stateadmin/stateadmin.go index 05a378037..4c8dd6fec 100644 --- a/internal/infrastructure/stateadmin/stateadmin.go +++ b/internal/infrastructure/stateadmin/stateadmin.go @@ -26,6 +26,7 @@ package stateadmin import ( + "database/sql" "errors" "fmt" "net/http" @@ -134,9 +135,15 @@ func connectTo(dbPath string) core.DBConnectFunc { func registerViews(app core.App) error { for _, v := range views { collection, err := app.FindCollectionByNameOrId(v.name) - if err != nil { + switch { + case errors.Is(err, sql.ErrNoRows): collection = core.NewViewCollection(v.name) - } else if !collection.IsView() { + case err != nil: + // A lookup that failed for any other reason must not be read as + // "absent": creating a second collection over a live one would + // be a schema change made on the strength of a transient error. + return fmt.Errorf("stateadmin: look up view %q: %w", v.name, err) + case !collection.IsView(): return fmt.Errorf("stateadmin: collection %q exists and is not a view", v.name) } collection.ViewQuery = v.query @@ -157,8 +164,13 @@ func registerViews(app core.App) error { // that is not there cannot be loosened again by a later PocketBase default. func removeDefaultAuthCollection(app core.App) error { collection, err := app.FindCollectionByNameOrId("users") - if err != nil { - return nil // already absent, including on every restart after the first + switch { + case errors.Is(err, sql.ErrNoRows): + // Already absent: every restart after the first, and any deployment + // whose PocketBase version stops shipping the default. + return nil + case err != nil: + return fmt.Errorf("stateadmin: look up default users collection: %w", err) } if err := app.Delete(collection); err != nil { return fmt.Errorf("stateadmin: remove default users collection: %w", err) From 57dbde07dadfe128fc5366d9218909197e1da31f Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Wed, 23 Sep 2026 01:27:34 +1000 Subject: [PATCH 06/20] feat(edastore): PocketBase-owned collections for event capture and bindings First slice of the persistence migration, taking the event-capture tables rather than the task lifecycle. These five tables hold no production rows and their feature (archie-core-t2db) is unfinished, so they migrate at zero data risk and the remaining work lands natively instead of being written against hand-rolled SQL and ported later. They are also the tables that actually want what PocketBase gives: inspecting a payload nobody has modelled yet, mapping fields against it, and editing a binding at runtime are the epic's acceptance criteria, not incidental conveniences. The task lifecycle deliberately stays on SQLite. tasks/transitions/events carry FIFO claim ordering, id-cursor pagination and the task-grant map, none of which gain anything from being hand-editable. binding_dispatches.task_id is the single seam and stays a plain integer addressing that store. Idempotency moves from INSERT OR IGNORE on a composite primary key to unique indexes, so the at-most-once ledgers are enforced by the schema rather than by each caller remembering the pattern. A save that fails for any other reason is returned rather than swallowed: treating every error as "already recorded" would turn a broken ledger into a silent at-least-once dispatcher. Adds tool_calls, which had no persistence at all. Tool calls were only written to the on-disk task log (internal/logging), readable as text but never queryable, filterable or watchable. --- internal/infrastructure/edastore/edastore.go | 111 +++++++++ internal/infrastructure/edastore/records.go | 211 ++++++++++++++++++ internal/infrastructure/edastore/schema.go | 191 ++++++++++++++++ .../infrastructure/edastore/schema_test.go | 129 +++++++++++ 4 files changed, 642 insertions(+) create mode 100644 internal/infrastructure/edastore/edastore.go create mode 100644 internal/infrastructure/edastore/records.go create mode 100644 internal/infrastructure/edastore/schema.go create mode 100644 internal/infrastructure/edastore/schema_test.go diff --git a/internal/infrastructure/edastore/edastore.go b/internal/infrastructure/edastore/edastore.go new file mode 100644 index 000000000..2adca1db6 --- /dev/null +++ b/internal/infrastructure/edastore/edastore.go @@ -0,0 +1,111 @@ +// Package edastore persists the event-capture and playbook-binding tables on +// PocketBase-owned collections. +// +// These are the tables behind the event-capture epic: an inbound webhook is +// captured whether or not anything understands it yet, an operator inspects +// the real payload, maps fields against it, and binds a matcher and mapping +// to a workflow. That loop wants a runtime-editable store with a live view of +// arriving payloads, which is what PocketBase collections give directly. +// +// Unlike the read-only surface in internal/infrastructure/stateadmin, these +// collections are authoritative and writable: the admin UI editing a binding +// IS the feature, not a hazard. The task lifecycle tables stay on SQLite -- +// they carry ordering, cursor and task-grant semantics that gain nothing from +// being hand-editable. binding_dispatches.task reaches them by plain id, which +// is the single seam between the two stores. +// +// Idempotency here is schema-enforced. The SQLite implementation relied on +// INSERT OR IGNORE against a composite primary key; a unique index does the +// same job without every caller having to remember the pattern. +package edastore + +import ( + "errors" + "fmt" + + "github.com/pocketbase/dbx" + "github.com/pocketbase/pocketbase" + "github.com/pocketbase/pocketbase/core" + _ "modernc.org/sqlite" +) + +// Collection names. They are unprefixed because this store owns its database; +// nothing else writes these tables. +const ( + CollCaptures = "captures" + CollMappings = "mappings" + CollBindings = "bindings" + CollBindingDispatches = "binding_dispatches" + CollPlaybookDispatches = "playbook_dispatches" + CollToolCalls = "tool_calls" +) + +// Binding lifecycle. A binding is only live once a human approves it, which is +// an acceptance criterion of the epic rather than a convention. +const ( + StatusDraft = "draft" + StatusApproved = "approved" +) + +var ( + // ErrBindingNotFound reports an operation naming a binding that is absent. + ErrBindingNotFound = errors.New("edastore: binding not found") + // ErrCaptureNotFound reports a capture that is absent or aged out. + ErrCaptureNotFound = errors.New("edastore: capture not found") +) + +// Collections returns every collection this store owns. +func Collections() []string { + return []string{ + CollCaptures, CollMappings, CollBindings, + CollBindingDispatches, CollPlaybookDispatches, CollToolCalls, + } +} + +// Config describes one store. +type Config struct { + // DBPath is the SQLite file PocketBase owns for these collections. + DBPath string + // DataDir holds PocketBase's auxiliary database and settings. + DataDir string +} + +// Store is the PocketBase-backed EDA persistence. +type Store struct{ app *pocketbase.PocketBase } + +// Open bootstraps the store and ensures its collections exist. +func Open(cfg Config) (*Store, error) { + if cfg.DBPath == "" || cfg.DataDir == "" { + return nil, errors.New("edastore: DBPath and DataDir are required") + } + app := pocketbase.NewWithConfig(pocketbase.Config{ + DefaultDataDir: cfg.DataDir, + HideStartBanner: true, + DBConnect: func(requested string) (*dbx.DB, error) { + if isDataDB(requested) { + requested = cfg.DBPath + } + return dbx.Open("sqlite", requested+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)") + }, + }) + if err := app.Bootstrap(); err != nil { + return nil, fmt.Errorf("edastore: bootstrap: %w", err) + } + if err := app.RunAllMigrations(); err != nil { + return nil, fmt.Errorf("edastore: migrations: %w", err) + } + if err := removeDefaultAuthCollection(app); err != nil { + return nil, err + } + if err := ensureCollections(app); err != nil { + return nil, err + } + return &Store{app: app}, nil +} + +// App exposes the PocketBase application so one process can serve this store +// and its admin UI from a single bootstrap. +func (s *Store) App() core.App { return s.app } + +// Close releases the store's database connections. +func (s *Store) Close() error { return s.app.ClearBootstrap() } diff --git a/internal/infrastructure/edastore/records.go b/internal/infrastructure/edastore/records.go new file mode 100644 index 000000000..f079c7830 --- /dev/null +++ b/internal/infrastructure/edastore/records.go @@ -0,0 +1,211 @@ +package edastore + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" + + "github.com/pocketbase/pocketbase/core" +) + +// Capture is one inbound event, stored whether or not anything understands it. +type Capture struct { + ID string + Source string + RemoteAddr string + ContentType string + Headers string + Body string + Authenticated bool +} + +// Binding ties a matcher and mapping to a workflow. +type Binding struct { + ID string + Name string + Source string + MappingID string + Workflow string + Owner string + Repo string + Version int64 + Status string + Secret string +} + +// Dispatch is one at-most-once binding dispatch ledger entry. +type Dispatch struct { + BindingID string + BindingVersion int64 + CaptureID string + // TaskID addresses the SQLite-owned task tables, which are not migrating. + TaskID int64 +} + +// PlaybookDispatch is one at-most-once playbook action ledger entry. +type PlaybookDispatch struct { + PlaybookID string + PlaybookVersion string + EventID string + ActionID string +} + +// InsertCapture stores one inbound event and returns its id. +func (s *Store) InsertCapture(_ context.Context, c Capture) (string, error) { + collection, err := s.app.FindCollectionByNameOrId(CollCaptures) + if err != nil { + return "", err + } + r := core.NewRecord(collection) + r.Set("source", c.Source) + r.Set("remote_addr", c.RemoteAddr) + r.Set("content_type", c.ContentType) + r.Set("headers", c.Headers) + r.Set("body", c.Body) + r.Set("authenticated", c.Authenticated) + if err := s.app.Save(r); err != nil { + return "", fmt.Errorf("edastore: insert capture: %w", err) + } + return r.Id, nil +} + +// Capture returns one stored event verbatim. +func (s *Store) Capture(_ context.Context, id string) (Capture, error) { + r, err := s.app.FindRecordById(CollCaptures, id) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return Capture{}, ErrCaptureNotFound + } + return Capture{}, err + } + return Capture{ + ID: r.Id, + Source: r.GetString("source"), + RemoteAddr: r.GetString("remote_addr"), + ContentType: r.GetString("content_type"), + Headers: r.GetString("headers"), + Body: r.GetString("body"), + Authenticated: r.GetBool("authenticated"), + }, nil +} + +// InsertBinding stores a new binding. It is always created as a draft: the +// epic requires explicit human approval before a binding can run anything. +func (s *Store) InsertBinding(_ context.Context, b Binding) (string, error) { + collection, err := s.app.FindCollectionByNameOrId(CollBindings) + if err != nil { + return "", err + } + r := core.NewRecord(collection) + r.Set("name", b.Name) + r.Set("source", b.Source) + r.Set("mapping", b.MappingID) + r.Set("workflow", b.Workflow) + r.Set("owner", b.Owner) + r.Set("repo", b.Repo) + r.Set("version", 1) + r.Set("status", StatusDraft) + r.Set("secret", b.Secret) + if err := s.app.Save(r); err != nil { + return "", fmt.Errorf("edastore: insert binding: %w", err) + } + return r.Id, nil +} + +// GetBinding returns one binding. +func (s *Store) GetBinding(_ context.Context, id string) (Binding, error) { + r, err := s.app.FindRecordById(CollBindings, id) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return Binding{}, ErrBindingNotFound + } + return Binding{}, err + } + return Binding{ + ID: r.Id, + Name: r.GetString("name"), + Source: r.GetString("source"), + MappingID: r.GetString("mapping"), + Workflow: r.GetString("workflow"), + Owner: r.GetString("owner"), + Repo: r.GetString("repo"), + Version: int64(r.GetInt("version")), + Status: r.GetString("status"), + Secret: r.GetString("secret"), + }, nil +} + +// ApproveBinding moves a draft binding to approved. It is the only way a +// binding becomes live. +func (s *Store) ApproveBinding(_ context.Context, id string) error { + r, err := s.app.FindRecordById(CollBindings, id) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return ErrBindingNotFound + } + return err + } + r.Set("status", StatusApproved) + r.Set("version", r.GetInt("version")+1) + if err := s.app.Save(r); err != nil { + return fmt.Errorf("edastore: approve binding: %w", err) + } + return nil +} + +// RecordDispatch writes one binding dispatch, reporting whether it was new. +// The (binding, capture) uniqueness is enforced by the collection's index, so +// a duplicate is refused by the schema rather than by the caller. +func (s *Store) RecordDispatch(_ context.Context, d Dispatch) (bool, error) { + collection, err := s.app.FindCollectionByNameOrId(CollBindingDispatches) + if err != nil { + return false, err + } + r := core.NewRecord(collection) + r.Set("binding", d.BindingID) + r.Set("binding_version", d.BindingVersion) + r.Set("capture", d.CaptureID) + r.Set("task_id", d.TaskID) + return savedOrDuplicate(s.app, r, "edastore: record dispatch") +} + +// RecordPlaybookDispatch writes one playbook action dispatch, reporting +// whether it was new. +func (s *Store) RecordPlaybookDispatch(_ context.Context, d PlaybookDispatch) (bool, error) { + collection, err := s.app.FindCollectionByNameOrId(CollPlaybookDispatches) + if err != nil { + return false, err + } + r := core.NewRecord(collection) + r.Set("playbook_id", d.PlaybookID) + r.Set("playbook_version", d.PlaybookVersion) + r.Set("event_id", d.EventID) + r.Set("action_id", d.ActionID) + return savedOrDuplicate(s.app, r, "edastore: record playbook dispatch") +} + +// savedOrDuplicate saves a ledger record, translating the unique-index +// violation into "already recorded" rather than an error. Anything else is a +// real failure and is returned: silently swallowing every save error here +// would turn a broken ledger into a silent at-least-once dispatcher. +func savedOrDuplicate(app core.App, r *core.Record, what string) (bool, error) { + err := app.Save(r) + switch { + case err == nil: + return true, nil + case isUniqueViolation(err): + return false, nil + default: + return false, fmt.Errorf("%s: %w", what, err) + } +} + +// isUniqueViolation reports whether err is the unique-index refusal. PocketBase +// surfaces it as a validation failure rather than a typed error, so the text is +// what there is to match on. +func isUniqueViolation(err error) bool { + text := strings.ToLower(err.Error()) + return strings.Contains(text, "unique") || strings.Contains(text, "already exists") +} diff --git a/internal/infrastructure/edastore/schema.go b/internal/infrastructure/edastore/schema.go new file mode 100644 index 000000000..f7c83f0ad --- /dev/null +++ b/internal/infrastructure/edastore/schema.go @@ -0,0 +1,191 @@ +package edastore + +import ( + "database/sql" + "errors" + "fmt" + "path/filepath" + + "github.com/pocketbase/pocketbase/core" +) + +// isDataDB reports whether PocketBase is asking for its main database, as +// opposed to the auxiliary one it keeps for logs and settings. +func isDataDB(path string) bool { return filepath.Base(path) == "data.db" } + +// removeDefaultAuthCollection deletes the open-signup "users" collection +// PocketBase creates on first bootstrap. This store has no end users, and its +// createRule is open, so leaving it would accept unauthenticated writes. +func removeDefaultAuthCollection(app core.App) error { + c, err := app.FindCollectionByNameOrId("users") + switch { + case errors.Is(err, sql.ErrNoRows): + return nil + case err != nil: + return fmt.Errorf("edastore: look up default users collection: %w", err) + } + if err := app.Delete(c); err != nil { + return fmt.Errorf("edastore: remove default users collection: %w", err) + } + return nil +} + +// ensureCollections creates each collection once and is safe to re-run: the +// store bootstraps on every process start. +func ensureCollections(app core.App) error { + for _, build := range []func(core.App) error{ + ensureCaptures, ensureMappings, ensureBindings, + ensureBindingDispatches, ensurePlaybookDispatches, ensureToolCalls, + } { + if err := build(app); err != nil { + return err + } + } + return nil +} + +// collection returns the named collection, or a new base collection when it +// does not exist yet. A lookup that failed for any other reason is an error: +// creating a second collection over a live one on a transient failure would be +// a schema change made for the wrong reason. +func collection(app core.App, name string) (*core.Collection, bool, error) { + c, err := app.FindCollectionByNameOrId(name) + switch { + case errors.Is(err, sql.ErrNoRows): + return core.NewBaseCollection(name), true, nil + case err != nil: + return nil, false, fmt.Errorf("edastore: look up collection %q: %w", name, err) + } + return c, false, nil +} + +func ensureCaptures(app core.App) error { + c, fresh, err := collection(app, CollCaptures) + if err != nil || !fresh { + return err + } + c.Fields.Add( + &core.TextField{Name: "source", Required: true}, + &core.TextField{Name: "remote_addr"}, + &core.TextField{Name: "content_type"}, + // Headers and body are stored verbatim. Field mapping is designed + // against the real payload, so nothing here may be normalised away. + &core.JSONField{Name: "headers", MaxSize: 1 << 20}, + &core.TextField{Name: "body", Max: 1 << 20}, + &core.BoolField{Name: "authenticated"}, + &core.AutodateField{Name: "received_at", OnCreate: true}, + ) + c.AddIndex("idx_captures_source", false, "source, received_at", "") + return save(app, c) +} + +func ensureMappings(app core.App) error { + c, fresh, err := collection(app, CollMappings) + if err != nil || !fresh { + return err + } + c.Fields.Add( + &core.TextField{Name: "name", Required: true}, + &core.TextField{Name: "source_hint"}, + &core.JSONField{Name: "fields", MaxSize: 1 << 18}, + &core.AutodateField{Name: "created_at", OnCreate: true}, + &core.AutodateField{Name: "updated_at", OnCreate: true, OnUpdate: true}, + ) + return save(app, c) +} + +func ensureBindings(app core.App) error { + c, fresh, err := collection(app, CollBindings) + if err != nil || !fresh { + return err + } + mappings, err := app.FindCollectionByNameOrId(CollMappings) + if err != nil { + return fmt.Errorf("edastore: bindings need mappings: %w", err) + } + c.Fields.Add( + &core.TextField{Name: "name", Required: true}, + &core.TextField{Name: "source", Required: true}, + &core.RelationField{Name: "mapping", CollectionId: mappings.Id, MaxSelect: 1}, + &core.TextField{Name: "workflow"}, + &core.TextField{Name: "owner"}, + &core.TextField{Name: "repo"}, + &core.NumberField{Name: "version"}, + // status carries the approval gate. It is deliberately not defaulted + // to anything live: a binding reaches "approved" only by ApproveBinding. + &core.SelectField{Name: "status", Values: []string{StatusDraft, StatusApproved}, MaxSelect: 1}, + &core.TextField{Name: "secret"}, + &core.AutodateField{Name: "created_at", OnCreate: true}, + &core.AutodateField{Name: "updated_at", OnCreate: true, OnUpdate: true}, + ) + c.AddIndex("idx_bindings_source_status", false, "source, status", "") + return save(app, c) +} + +func ensureBindingDispatches(app core.App) error { + c, fresh, err := collection(app, CollBindingDispatches) + if err != nil || !fresh { + return err + } + c.Fields.Add( + &core.TextField{Name: "binding", Required: true}, + &core.NumberField{Name: "binding_version"}, + &core.TextField{Name: "capture", Required: true}, + // task_id is the seam to the SQLite-owned task tables. It stays a + // plain integer id rather than a relation precisely because the task + // lifecycle is not migrating. + &core.NumberField{Name: "task_id"}, + &core.AutodateField{Name: "dispatched_at", OnCreate: true}, + ) + // At-most-once, enforced by the schema rather than by every caller + // remembering INSERT OR IGNORE. + c.AddIndex("idx_binding_dispatch_once", true, "binding, capture", "") + return save(app, c) +} + +func ensurePlaybookDispatches(app core.App) error { + c, fresh, err := collection(app, CollPlaybookDispatches) + if err != nil || !fresh { + return err + } + c.Fields.Add( + &core.TextField{Name: "playbook_id", Required: true}, + &core.TextField{Name: "playbook_version", Required: true}, + &core.TextField{Name: "event_id", Required: true}, + &core.TextField{Name: "action_id", Required: true}, + &core.AutodateField{Name: "dispatched_at", OnCreate: true}, + ) + c.AddIndex("idx_playbook_dispatch_once", true, + "playbook_id, playbook_version, event_id, action_id", "") + return save(app, c) +} + +// ensureToolCalls adds persistence that did not exist before. Tool calls were +// only ever written to the on-disk task log (internal/logging), so they could +// be read back as text but never queried, filtered or watched. As a collection +// they become all three. +func ensureToolCalls(app core.App) error { + c, fresh, err := collection(app, CollToolCalls) + if err != nil || !fresh { + return err + } + c.Fields.Add( + &core.NumberField{Name: "task_id"}, + &core.NumberField{Name: "attempt"}, + &core.TextField{Name: "tool", Required: true}, + &core.JSONField{Name: "args", MaxSize: 1 << 18}, + &core.JSONField{Name: "result", MaxSize: 1 << 18}, + &core.TextField{Name: "error"}, + &core.NumberField{Name: "duration_ms"}, + &core.AutodateField{Name: "called_at", OnCreate: true}, + ) + c.AddIndex("idx_tool_calls_task", false, "task_id, attempt", "") + return save(app, c) +} + +func save(app core.App, c *core.Collection) error { + if err := app.Save(c); err != nil { + return fmt.Errorf("edastore: create collection %q: %w", c.Name, err) + } + return nil +} diff --git a/internal/infrastructure/edastore/schema_test.go b/internal/infrastructure/edastore/schema_test.go new file mode 100644 index 000000000..242d64534 --- /dev/null +++ b/internal/infrastructure/edastore/schema_test.go @@ -0,0 +1,129 @@ +package edastore_test + +import ( + "errors" + "path/filepath" + "testing" + + "github.com/pocketbase/pocketbase/core" + + "github.com/samcharles93/archie-core/internal/infrastructure/edastore" +) + +func newStore(t *testing.T) *edastore.Store { + t.Helper() + dir := t.TempDir() + st, err := edastore.Open(edastore.Config{ + DBPath: filepath.Join(dir, "eda.sqlite"), + DataDir: filepath.Join(dir, "pb_data"), + }) + if err != nil { + t.Fatalf("edastore.Open() error = %v", err) + } + t.Cleanup(func() { _ = st.Close() }) + return st +} + +// TestCollectionsAreCreated pins that every EDA collection exists and is a +// base collection, not a view: unlike the read-only operator surface, these +// are the authoritative tables and the admin UI is expected to edit them. +func TestCollectionsAreCreated(t *testing.T) { + st := newStore(t) + for _, name := range edastore.Collections() { + c, err := st.App().FindCollectionByNameOrId(name) + if err != nil { + t.Fatalf("collection %q missing: %v", name, err) + } + if c.IsView() { + t.Errorf("collection %q is a view; EDA tables are authoritative and must be writable", name) + } + } +} + +// TestBindingDispatchIsIdempotent pins the at-most-once ledger. The SQLite +// implementation got this from INSERT OR IGNORE on a composite primary key +// (bindings.go:365); here it must come from a unique index, so a duplicate is +// refused by the schema rather than by the caller remembering to ignore it. +func TestBindingDispatchIsIdempotent(t *testing.T) { + st := newStore(t) + first, err := st.RecordDispatch(t.Context(), edastore.Dispatch{ + BindingID: "b1", BindingVersion: 1, CaptureID: "c1", TaskID: 42, + }) + if err != nil { + t.Fatalf("first RecordDispatch() error = %v", err) + } + if !first { + t.Fatal("first RecordDispatch() = false, want true: a fresh dispatch must be recorded") + } + second, err := st.RecordDispatch(t.Context(), edastore.Dispatch{ + BindingID: "b1", BindingVersion: 1, CaptureID: "c1", TaskID: 99, + }) + if err != nil { + t.Fatalf("second RecordDispatch() error = %v", err) + } + if second { + t.Error("second RecordDispatch() = true, want false: (binding, capture) is an at-most-once ledger") + } +} + +// TestPlaybookDispatchIsIdempotent pins the same at-most-once rule on the +// four-part playbook key (playbook, version, event, action). +func TestPlaybookDispatchIsIdempotent(t *testing.T) { + st := newStore(t) + key := edastore.PlaybookDispatch{PlaybookID: "p1", PlaybookVersion: "v1", EventID: "e1", ActionID: "a1"} + if ok, err := st.RecordPlaybookDispatch(t.Context(), key); err != nil || !ok { + t.Fatalf("first RecordPlaybookDispatch() = %v, %v; want true, nil", ok, err) + } + if ok, err := st.RecordPlaybookDispatch(t.Context(), key); err != nil || ok { + t.Fatalf("second RecordPlaybookDispatch() = %v, %v; want false, nil", ok, err) + } +} + +// TestBindingStartsAsDraft pins the approval gate from the epic's acceptance +// criteria: "A binding cannot go live without explicit human approval." A +// binding created without a status must not default to anything live. +func TestBindingStartsAsDraft(t *testing.T) { + st := newStore(t) + id, err := st.InsertBinding(t.Context(), edastore.Binding{Name: "n", Source: "sentry", Workflow: "implement"}) + if err != nil { + t.Fatalf("InsertBinding() error = %v", err) + } + b, err := st.GetBinding(t.Context(), id) + if err != nil { + t.Fatalf("GetBinding() error = %v", err) + } + if b.Status != edastore.StatusDraft { + t.Errorf("new binding status = %q, want %q: a binding must not be live before approval", b.Status, edastore.StatusDraft) + } +} + +// TestApproveRejectsUnknownBinding pins the sentinel the gRPC layer maps. +func TestApproveRejectsUnknownBinding(t *testing.T) { + st := newStore(t) + if err := st.ApproveBinding(t.Context(), "nosuchbinding"); !errors.Is(err, edastore.ErrBindingNotFound) { + t.Errorf("ApproveBinding(unknown) error = %v, want ErrBindingNotFound", err) + } +} + +// TestCaptureRoundTripsPayload pins the epic's core requirement: an event +// arrives with a payload nobody has modelled yet and must be readable back +// verbatim so an operator can map fields against it. +func TestCaptureRoundTripsPayload(t *testing.T) { + st := newStore(t) + body := `{"action":"created","issue":{"number":7,"title":"it broke"}}` + id, err := st.InsertCapture(t.Context(), edastore.Capture{ + Source: "sentry", Body: body, Headers: `{"X-Hook":"1"}`, ContentType: "application/json", + }) + if err != nil { + t.Fatalf("InsertCapture() error = %v", err) + } + got, err := st.Capture(t.Context(), id) + if err != nil { + t.Fatalf("Capture() error = %v", err) + } + if got.Body != body { + t.Errorf("Body round-trip = %q, want %q: the raw payload is what field mapping is designed against", got.Body, body) + } +} + +var _ = core.Collection{} From 2e1960dfe004344feb6e70da6f2c78b3e8716bca Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Wed, 23 Sep 2026 01:37:12 +1000 Subject: [PATCH 07/20] refactor(scheduling): delete the legacy file-backed cronstore The schedules feature no longer runs on it: jobs live in the state store as the schedules control-plane document, read and written through the control-plane client (scheduleResourceStore), and the legacy jobs.json migration into that document has run its course. What remained of internal/infrastructure/cronstore was 1,600 lines of file store, OS-level cross-process locking and schema-versioned JSON envelope -- plus the schedule vocabulary held hostage inside it. The vocabulary moves to internal/domain/scheduling, which is where migration-decisions already said a schedules vocabulary belongs: a domain vocabulary package must not import infrastructure. JobSpec, Target, Payload, the kind constants, Schedule and Duration move unchanged -- same wire shape, same JSON tags -- with the two sentinel errors callers matched outside the package (ErrJobNotFound, ErrScheduleUnsupported) and ErrInvalidSpec, which the schedule arithmetic itself raises. The store's own sentinels die with the store. The legacy migration path (migrateLegacySchedules, readLegacyJobs) goes with it: its only dependency was cronstore.Open, and its work is done. crondelivery keeps its consumer-owned contracts; the compile-time assertions that pinned *cronstore.Store move to the production implementation, scheduleResourceStore, in internal/app/archied. The package's fixtures swap the real file store for an in-memory one with the same Get/MarkRun/Due semantics, so the run-recording acceptance test still asserts against real schedule arithmetic. A future storage backend (PocketBase) implements the same vocabulary. --- docs/architecture/migration-decisions.md | 7 +- .../archied/migrate_legacy_schedules_test.go | 206 ------ internal/app/archied/scheduling.go | 23 +- internal/app/archied/state_store.go | 82 +-- internal/app/controlplane/client.go | 10 +- internal/app/controlplane/schedules.go | 11 +- internal/app/controlplane/schedules_test.go | 6 +- internal/app/controlplane/schema.go | 4 +- internal/domain/scheduling/jobspec.go | 112 +++ .../scheduling}/schedule.go | 2 +- .../infrastructure/crondelivery/contract.go | 31 +- .../crondelivery/helpers_test.go | 129 +++- .../infrastructure/crondelivery/router.go | 9 +- .../crondelivery/router_test.go | 69 +- .../crondelivery/runner_test.go | 5 +- internal/infrastructure/cronstore/doc.go | 41 -- internal/infrastructure/cronstore/errors.go | 39 - internal/infrastructure/cronstore/kind.go | 24 - .../infrastructure/cronstore/kind_test.go | 148 ---- .../infrastructure/cronstore/lock_unix.go | 68 -- .../infrastructure/cronstore/lock_windows.go | 53 -- .../infrastructure/cronstore/process_lock.go | 33 - .../cronstore/schedule_duration_test.go | 91 --- internal/infrastructure/cronstore/store.go | 665 ------------------ .../cronstore/store_review_test.go | 485 ------------- .../infrastructure/cronstore/store_test.go | 512 -------------- 26 files changed, 297 insertions(+), 2568 deletions(-) delete mode 100644 internal/app/archied/migrate_legacy_schedules_test.go create mode 100644 internal/domain/scheduling/jobspec.go rename internal/{infrastructure/cronstore => domain/scheduling}/schedule.go (99%) delete mode 100644 internal/infrastructure/cronstore/doc.go delete mode 100644 internal/infrastructure/cronstore/errors.go delete mode 100644 internal/infrastructure/cronstore/kind.go delete mode 100644 internal/infrastructure/cronstore/kind_test.go delete mode 100644 internal/infrastructure/cronstore/lock_unix.go delete mode 100644 internal/infrastructure/cronstore/lock_windows.go delete mode 100644 internal/infrastructure/cronstore/process_lock.go delete mode 100644 internal/infrastructure/cronstore/schedule_duration_test.go delete mode 100644 internal/infrastructure/cronstore/store.go delete mode 100644 internal/infrastructure/cronstore/store_review_test.go delete mode 100644 internal/infrastructure/cronstore/store_test.go diff --git a/docs/architecture/migration-decisions.md b/docs/architecture/migration-decisions.md index 0e4a4a660..326c1f348 100644 --- a/docs/architecture/migration-decisions.md +++ b/docs/architecture/migration-decisions.md @@ -264,9 +264,10 @@ around is no longer a boundary. returns and validates `workflow.ExecutionSettings`, so any package holding it links the workflow domain and therefore `internal/agentexec`. This is why the client cannot move as one package -- see open question 2. -- The schedules document, whose type is `cronstore.JobSpec` in - `internal/infrastructure`. A domain vocabulary package may not import - infrastructure. +- The schedules document, whose type is `scheduling.JobSpec` in + `internal/domain/scheduling` -- the vocabulary moved there when the legacy + file-backed cronstore was deleted, which is also what satisfies the rule + that a domain vocabulary package may not import infrastructure. - The schema derivation, which describes the documents the server advertises. **Placement as landed (2026-09-22, `archie-core-1ng1`).** One new package, diff --git a/internal/app/archied/migrate_legacy_schedules_test.go b/internal/app/archied/migrate_legacy_schedules_test.go deleted file mode 100644 index dcf35ddeb..000000000 --- a/internal/app/archied/migrate_legacy_schedules_test.go +++ /dev/null @@ -1,206 +0,0 @@ -package archied - -import ( - "encoding/json" - "log/slog" - "os" - "path/filepath" - "testing" - "time" - - "github.com/samcharles93/archie-core/internal/app/controlplane" - controlpb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1" - "github.com/samcharles93/archie-core/internal/infrastructure/cronstore" - "github.com/samcharles93/archie-core/internal/store" -) - -// migrationServer builds the validating side exactly as RunStateStore composes -// it: a store over a throwaway file and openStateStoreControlPlane, the root -// helper that registers the shared step vocabulary. -func migrationServer(t *testing.T) (*controlplane.Server, *store.Store) { - t.Helper() - st, err := store.Open(t.Context(), filepath.Join(t.TempDir(), "archie.db-tasks.sqlite")) - if err != nil { - t.Fatalf("open store: %v", err) - } - t.Cleanup(func() { _ = st.Close() }) - server, err := openStateStoreControlPlane(st) - if err != nil { - t.Fatalf("open control plane: %v", err) - } - return server, st -} - -// seedSchedules mirrors RunStateStore's ImportConfig: the schedules resource -// starts as the empty seed at version 1, which is the state the legacy -// migration runs against. -func seedSchedules(t *testing.T, control *controlplane.Server) int64 { - t.Helper() - seed, err := json.Marshal([]cronstore.JobSpec{}) - if err != nil { - t.Fatalf("encode empty schedules seed: %v", err) - } - response, err := control.Command(t.Context(), &controlpb.CommandRequest{ - Kind: controlplane.SchedulesKind, Command: "replace", ValueJson: seed, - ExpectedVersion: 0, Actor: "system:migration", Source: "legacy-config", RequestId: "import:schedules", - }) - if err != nil { - t.Fatalf("seed schedules: %v", err) - } - return response.GetResource().GetVersion() -} - -// writeLegacyJobs writes a legacy cronstore file at the layout -// migrateLegacySchedules reads: /cron/jobs.json, in the -// store's own on-disk envelope shape (schema v2). -func writeLegacyJobs(t *testing.T, dir string, jobs []cronstore.JobSpec) string { - t.Helper() - data, err := json.Marshal(struct { - SchemaVersion int `json:"schema_version"` - Jobs []cronstore.JobSpec `json:"jobs"` - }{SchemaVersion: 2, Jobs: jobs}) - if err != nil { - t.Fatalf("encode fixture: %v", err) - } - return writeLegacyFile(t, dir, string(data)) -} - -func writeLegacyFile(t *testing.T, dir, content string) string { - t.Helper() - path := filepath.Join(dir, "cron", "jobs.json") - if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { - t.Fatalf("create legacy cron directory: %v", err) - } - if err := os.WriteFile(path, []byte(content), 0o644); err != nil { - t.Fatalf("write legacy jobs file: %v", err) - } - return path -} - -// legacyJob is a schema-v1-shaped job the old daemon's cron engine wrote. -// An empty Kind means chat, the pre-field default (cronstore.KindChat). -func legacyJob(id, kind string, interval time.Duration) cronstore.JobSpec { - stamped := time.Date(2026, 9, 1, 9, 0, 0, 0, time.UTC) - return cronstore.JobSpec{ - ID: id, Kind: kind, Detail: "legacy " + id, - Schedule: cronstore.Schedule{Kind: cronstore.ScheduleInterval, Interval: cronstore.Duration(interval)}, - NextRun: stamped, Created: stamped, Updated: stamped, - } -} - -// storedScheduleIDs decodes the schedules resource into the job IDs it holds. -func storedScheduleIDs(t *testing.T, st *store.Store) []string { - t.Helper() - resource, err := st.Resource(t.Context(), controlplane.SchedulesKind) - if err != nil { - t.Fatalf("read schedules resource: %v", err) - } - var jobs []cronstore.JobSpec - if err := json.Unmarshal(resource.Value, &jobs); err != nil { - t.Fatalf("decode schedules resource: %v", err) - } - ids := make([]string, 0, len(jobs)) - for _, job := range jobs { - ids = append(ids, job.ID) - } - return ids -} - -// TestMigrateLegacySchedulesMovesOnlyWorkflowJobs: a legacy file holding a -// chat job (the pre-field default, kind omitted) and a workflow job migrates -// the workflow job and skips -- never refuses -- the chat one. The current -// behaviour replaces the whole list and lets the schedules validator fail the -// State Store on the chat job, taking every process that dials it down with -// it (archie-core-q6vw). -func TestMigrateLegacySchedulesMovesOnlyWorkflowJobs(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "archie.db-tasks.sqlite") - writeLegacyJobs(t, dir, []cronstore.JobSpec{ - legacyJob("chat-daily", "", 24*time.Hour), - legacyJob("tdd-hourly", cronstore.KindWorkflow, time.Hour), - }) - server, st := migrationServer(t) - version := seedSchedules(t, server) - - migrateLegacySchedules(t.Context(), server, dbPath, version, discardLog()) - got := storedScheduleIDs(t, st) - if len(got) != 1 || got[0] != "tdd-hourly" { - t.Fatalf("schedules resource holds %v, want only the migratable workflow job", got) - } - if _, err := os.Stat(path_migrated(dir)); err != nil { - t.Fatalf("migrated legacy file not marked (jobs.json.migrated): %v", err) - } - if _, err := os.Stat(filepath.Join(dir, "cron", "jobs.json")); !os.IsNotExist(err) { - t.Fatalf("legacy file still at its original path after a successful migration") - } -} - -// path_migrated is where a completed migration leaves the legacy file. -func path_migrated(dir string) string { - return filepath.Join(dir, "cron", "jobs.json.migrated") -} - -// TestMigrateLegacySchedulesKeepsStoreRunningWhenNothingIsMigratable: a -// legacy file holding only chat jobs leaves the schedules resource untouched -// and reports -- never fails -- the skip. The store's whole job is to come up. -func TestMigrateLegacySchedulesKeepsStoreRunningWhenNothingIsMigratable(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "archie.db-tasks.sqlite") - path := writeLegacyJobs(t, dir, []cronstore.JobSpec{ - legacyJob("chat-daily", "", 24*time.Hour), - legacyJob("chat-weekly", "chat", 7*24*time.Hour), - }) - server, st := migrationServer(t) - version := seedSchedules(t, server) - - migrateLegacySchedules(t.Context(), server, dbPath, version, discardLog()) - if _, err := os.Stat(path); err != nil { - t.Fatalf("legacy file removed though nothing was migrated: %v", err) - } - if got := storedScheduleIDs(t, st); len(got) != 0 { - t.Fatalf("schedules resource holds %v, want the untouched empty seed", got) - } -} - -// TestMigrateLegacySchedulesKeepsStoreRunningOnUnreadableLegacyStore: a -// corrupt legacy file is a diagnosis, not a boot blocker. The store must come -// up on the empty seed and the file must stay in place for the operator. -func TestMigrateLegacySchedulesKeepsStoreRunningOnUnreadableLegacyStore(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "archie.db-tasks.sqlite") - path := writeLegacyFile(t, dir, "{not json") - server, _ := migrationServer(t) - version := seedSchedules(t, server) - - migrateLegacySchedules(t.Context(), server, dbPath, version, discardLog()) - if _, err := os.Stat(path); err != nil { - t.Fatalf("legacy file removed though it was never read: %v", err) - } -} - -// TestMigrateLegacySchedulesLeavesTheFileWhenTheReplaceIsRefused: even a -// workflow job the schedules validator refuses is skipped and retried on the -// next start, the way ImportConfig skips a refused seed -- never fatal. -func TestMigrateLegacySchedulesLeavesTheFileWhenTheReplaceIsRefused(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "archie.db-tasks.sqlite") - path := writeLegacyJobs(t, dir, []cronstore.JobSpec{ - legacyJob("broken-interval", cronstore.KindWorkflow, 0), // interval schedule with no interval - }) - server, st := migrationServer(t) - version := seedSchedules(t, server) - - migrateLegacySchedules(t.Context(), server, dbPath, version, discardLog()) - if _, err := os.Stat(path); err != nil { - t.Fatalf("legacy file removed though the replace never landed: %v", err) - } - if got := storedScheduleIDs(t, st); len(got) != 0 { - t.Fatalf("schedules resource holds %v, want the untouched empty seed", got) - } -} - -// discardLog returns a logger that drops every record: these tests pin the -// migration's effects on the resource and the file, not its reporting. -func discardLog() *slog.Logger { - return slog.New(slog.DiscardHandler) -} diff --git a/internal/app/archied/scheduling.go b/internal/app/archied/scheduling.go index 0e10eacd3..a4fa4df0e 100644 --- a/internal/app/archied/scheduling.go +++ b/internal/app/archied/scheduling.go @@ -14,7 +14,6 @@ import ( "github.com/samcharles93/archie-core/internal/infrastructure/configuration" "github.com/samcharles93/archie-core/internal/infrastructure/crondelivery" "github.com/samcharles93/archie-core/internal/infrastructure/cronevents" - "github.com/samcharles93/archie-core/internal/infrastructure/cronstore" ) // schedulingConfig translates external input into the domain-owned engine @@ -75,7 +74,7 @@ func (b *boot) setupScheduling() error { if err != nil { return fmt.Errorf("scheduling: build workflow runner: %w", err) } - router, err := crondelivery.NewRouter(store, map[string]scheduling.Runner{cronstore.KindWorkflow: workflowRunner}, cfg.Events) + router, err := crondelivery.NewRouter(store, map[string]scheduling.Runner{scheduling.KindWorkflow: workflowRunner}, cfg.Events) if err != nil { return fmt.Errorf("scheduling: build router: %w", err) } @@ -93,17 +92,27 @@ type scheduleResourceStore struct { client *controlplane.Client } -func (s scheduleResourceStore) Get(ctx context.Context, id string) (cronstore.JobSpec, bool, error) { +// The schedules document behind the control plane is the production +// RouterStore: it resolves a job id to its spec and records a completed run. +// The assertions this replaces used to pin the retired file-backed store; +// crondelivery's contract comments name this type as the implementation. +var ( + _ crondelivery.SpecLookup = scheduleResourceStore{} + _ crondelivery.RunRecorder = scheduleResourceStore{} + _ crondelivery.RouterStore = scheduleResourceStore{} +) + +func (s scheduleResourceStore) Get(ctx context.Context, id string) (scheduling.JobSpec, bool, error) { jobs, _, err := s.client.Schedules(ctx) if err != nil { - return cronstore.JobSpec{}, false, err + return scheduling.JobSpec{}, false, err } for _, job := range jobs { if job.ID == id { return job, true, nil } } - return cronstore.JobSpec{}, false, nil + return scheduling.JobSpec{}, false, nil } func (s scheduleResourceStore) Due(ctx context.Context, now time.Time) ([]scheduling.Job, error) { @@ -132,7 +141,7 @@ func (s scheduleResourceStore) MarkRun(ctx context.Context, id string, runAt tim continue } found = true - if jobs[index].Schedule.Resolved().Kind == cronstore.ScheduleOnce { + if jobs[index].Schedule.Resolved().Kind == scheduling.ScheduleOnce { jobs = append(jobs[:index], jobs[index+1:]...) break } @@ -146,7 +155,7 @@ func (s scheduleResourceStore) MarkRun(ctx context.Context, id string, runAt tim break } if !found { - return fmt.Errorf("%w: id %q", cronstore.ErrJobNotFound, id) + return fmt.Errorf("%w: id %q", scheduling.ErrJobNotFound, id) } _, err = s.client.ReplaceSchedules(ctx, jobs, version, "system:scheduler", "scheduler", fmt.Sprintf("schedule-run:%s:%d", id, runAt.UnixNano())) if err == nil { diff --git a/internal/app/archied/state_store.go b/internal/app/archied/state_store.go index 1fe82d3ef..7b6ca1498 100644 --- a/internal/app/archied/state_store.go +++ b/internal/app/archied/state_store.go @@ -13,10 +13,8 @@ import ( "context" "encoding/json" "fmt" - "log/slog" "net" "net/http" - "os" "path/filepath" "time" @@ -24,11 +22,9 @@ import ( "github.com/samcharles93/archie-core/internal/app/controlplane" "github.com/samcharles93/archie-core/internal/config" - controlpb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1" "github.com/samcharles93/archie-core/internal/domain/health" "github.com/samcharles93/archie-core/internal/domain/identity" "github.com/samcharles93/archie-core/internal/domain/storecontract" - "github.com/samcharles93/archie-core/internal/infrastructure/cronstore" "github.com/samcharles93/archie-core/internal/infrastructure/readiness" "github.com/samcharles93/archie-core/internal/infrastructure/stateadmin" "github.com/samcharles93/archie-core/internal/infrastructure/staterpc" @@ -112,12 +108,11 @@ func RunStateStore(ctx context.Context, options StateStoreOptions) error { if err != nil { return err } - versions, skipped, err := control.ImportConfig(ctx, b.cfg) + _, skipped, err := control.ImportConfig(ctx, b.cfg) if err != nil { return fmt.Errorf("import control-plane resources: %w", err) } b.reportUnseededResources(skipped) - migrateLegacySchedules(ctx, control, b.cfg.DBPath, versions[controlplane.SchedulesKind], b.log) token := options.Token if token == "" { @@ -191,81 +186,6 @@ func openStateStoreControlPlane(resources controlplane.ResourceStore) (*controlp return server, nil } -// migrateLegacySchedules carries a legacy cronstore file -- the old daemon's -// /cron/jobs.json -- into the schedules resource, once, on a fresh -// seed. It moves only the jobs the schedules resource can hold: workflow -// jobs, the one delivery the router maps. Chat and unrecognised kinds were -// already dead deliveries in the old engine (the delivery router maps -// KindWorkflow alone), so they are reported and left in the legacy file -// rather than being allowed to refuse the replace the whole store depends on. -// -// Nothing here is fatal. The migration is a legacy-data courtesy: an -// unreadable or refused sidecar is logged and left in place for the next -// start, the way ImportConfig skips a seed the resource validator refuses -- -// a value this process cannot use must not stop the one process every other -// process dials (docs/prds/runtime-control-plane.md, "Bootstrap, migration, -// and recovery"). A successful replace is marked by renaming the file to -// jobs.json.migrated, so the whole legacy list -- the skipped jobs included -// -- stays on disk for the operator; until the rename lands, the replaying -// request ID keeps the replace idempotent across restarts. -func migrateLegacySchedules(ctx context.Context, control *controlplane.Server, dbPath string, version int64, log *slog.Logger) { - if version != 1 { - return - } - path := filepath.Join(filepath.Dir(dbPath), "cron", "jobs.json") - if _, err := os.Stat(path); err != nil { - if os.IsNotExist(err) { - return - } - log.Warn("legacy cron store not readable; schedules migration skipped and retried on the next start", "path", path, "err", err) - return - } - jobs, err := readLegacyJobs(ctx, path) - if err != nil { - log.Error("legacy cron store unreadable; schedules migration skipped and retried on the next start", "path", path, "err", err) - return - } - migratable := make([]cronstore.JobSpec, 0, len(jobs)) - for _, job := range jobs { - if job.Kind == cronstore.KindWorkflow { - migratable = append(migratable, job) - continue - } - log.Warn("legacy cron job is not a workflow job; the schedules resource does not carry it and it stays in the legacy file", "id", job.ID, "kind", job.Kind) - } - if len(migratable) == 0 { - log.Info("legacy cron store holds no workflow jobs; nothing to migrate", "path", path, "jobs", len(jobs)) - return - } - value, err := json.Marshal(migratable) - if err != nil { - log.Error("encode legacy cron jobs; schedules migration skipped and retried on the next start", "path", path, "err", err) - return - } - if _, err := control.Command(ctx, &controlpb.CommandRequest{Kind: controlplane.SchedulesKind, Command: "replace", ValueJson: value, ExpectedVersion: 1, Actor: "system:migration", Source: "legacy-cronstore", RequestId: "import:legacy-schedules"}); err != nil { - log.Error("legacy cron jobs do not satisfy the schedules resource; migration skipped and retried on the next start", "path", path, "err", err) - return - } - if err := os.Rename(path, path+".migrated"); err != nil { - log.Error("mark the migrated legacy cron file; migration retried on the next start", "path", path, "err", err) - } -} - -// readLegacyJobs reads the legacy cronstore file and closes its handle, -// reporting whichever failure came first. -func readLegacyJobs(ctx context.Context, path string) ([]cronstore.JobSpec, error) { - legacy, err := cronstore.Open(path) - if err != nil { - return nil, err - } - jobs, listErr := legacy.List(ctx) - closeErr := legacy.Close() - if listErr != nil { - return nil, listErr - } - return jobs, closeErr -} - // openStateStore opens the single task-store SQLite file exactly once for // service ownership. It resolves secrets (for the binding cipher) but does // NOT open gateway chat sessions -- those live in the separate archie-gateway diff --git a/internal/app/controlplane/client.go b/internal/app/controlplane/client.go index 58b1564c1..aa411513e 100644 --- a/internal/app/controlplane/client.go +++ b/internal/app/controlplane/client.go @@ -11,9 +11,9 @@ import ( pb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1" "github.com/samcharles93/archie-core/internal/domain/agent" + "github.com/samcharles93/archie-core/internal/domain/scheduling" "github.com/samcharles93/archie-core/internal/domain/workflow" "github.com/samcharles93/archie-core/internal/infrastructure/controlplanerpc" - "github.com/samcharles93/archie-core/internal/infrastructure/cronstore" ) // Client is the control-plane client for every resource that does not resolve @@ -25,7 +25,7 @@ import ( // Client is the control-plane client a process that serves the store uses. It // embeds the Messaging Service's client -- the generic resource read path and // the channel-settings projection -- and adds the resource-specific reads and -// watches, which need the workflow and cronstore engines that such a process +// watches, which need the workflow and scheduling engines that such a process // already links (archie-core-1ng1). type Client struct { *controlplanerpc.Client @@ -213,19 +213,19 @@ func sendUpdate[T any](ctx context.Context, out chan<- T, update T) bool { } } -func (c *Client) Schedules(ctx context.Context) ([]cronstore.JobSpec, int64, error) { +func (c *Client) Schedules(ctx context.Context) ([]scheduling.JobSpec, int64, error) { response, err := c.rpc.Query(ctx, &pb.QueryRequest{Kind: SchedulesKind}) if err != nil { return nil, 0, controlplanerpc.ClientError(err) } - var jobs []cronstore.JobSpec + var jobs []scheduling.JobSpec if err := json.Unmarshal(response.Resource.ValueJson, &jobs); err != nil { return nil, 0, err } return jobs, response.Resource.Version, nil } -func (c *Client) ReplaceSchedules(ctx context.Context, jobs []cronstore.JobSpec, expectedVersion int64, actor, source, requestID string) (int64, error) { +func (c *Client) ReplaceSchedules(ctx context.Context, jobs []scheduling.JobSpec, expectedVersion int64, actor, source, requestID string) (int64, error) { value, err := json.Marshal(jobs) if err != nil { return 0, err diff --git a/internal/app/controlplane/schedules.go b/internal/app/controlplane/schedules.go index bbc0fe2ff..4e8e15e3b 100644 --- a/internal/app/controlplane/schedules.go +++ b/internal/app/controlplane/schedules.go @@ -8,7 +8,6 @@ import ( "github.com/samcharles93/archie-core/internal/config" "github.com/samcharles93/archie-core/internal/domain/scheduling" - "github.com/samcharles93/archie-core/internal/infrastructure/cronstore" ) const SchedulesKind = "schedules" @@ -17,9 +16,9 @@ func scheduleDefinition() Definition { return Definition{ Kind: SchedulesKind, Title: "Schedules", - Document: []cronstore.JobSpec{}, + Document: []scheduling.JobSpec{}, ApplyMode: "live", - Seed: func(config.Config) any { return []cronstore.JobSpec{} }, + Seed: func(config.Config) any { return []scheduling.JobSpec{} }, Validate: func(input []byte) error { return validateAs(input, validateSchedules) }, @@ -27,7 +26,7 @@ func scheduleDefinition() Definition { } } -func validateSchedules(jobs []cronstore.JobSpec) error { +func validateSchedules(jobs []scheduling.JobSpec) error { seen := make(map[string]struct{}, len(jobs)) for _, job := range jobs { if strings.TrimSpace(job.ID) == "" { @@ -37,7 +36,7 @@ func validateSchedules(jobs []cronstore.JobSpec) error { return fmt.Errorf("duplicate schedule %q", job.ID) } seen[job.ID] = struct{}{} - if job.Kind != cronstore.KindWorkflow { + if job.Kind != scheduling.KindWorkflow { return fmt.Errorf("schedule %q has unsupported kind %q", job.ID, job.Kind) } if err := job.Schedule.Validate(); err != nil { @@ -51,7 +50,7 @@ func validateSchedules(jobs []cronstore.JobSpec) error { } func normalizeSchedules(input []byte) ([]byte, error) { - var jobs []cronstore.JobSpec + var jobs []scheduling.JobSpec if err := json.Unmarshal(input, &jobs); err != nil { return nil, err } diff --git a/internal/app/controlplane/schedules_test.go b/internal/app/controlplane/schedules_test.go index 3a7d20f42..8a7ae612a 100644 --- a/internal/app/controlplane/schedules_test.go +++ b/internal/app/controlplane/schedules_test.go @@ -5,14 +5,14 @@ import ( "testing" "time" - "github.com/samcharles93/archie-core/internal/infrastructure/cronstore" + "github.com/samcharles93/archie-core/internal/domain/scheduling" ) func TestScheduleDefinitionComputesFirstRunAndRejectsDuplicateIDs(t *testing.T) { t.Parallel() definition := scheduleDefinition() - jobs := []cronstore.JobSpec{{ID: "daily", Kind: cronstore.KindWorkflow, Schedule: cronstore.Schedule{Kind: cronstore.ScheduleInterval, Interval: cronstore.Duration(time.Hour)}}} + jobs := []scheduling.JobSpec{{ID: "daily", Kind: scheduling.KindWorkflow, Schedule: scheduling.Schedule{Kind: scheduling.ScheduleInterval, Interval: scheduling.Duration(time.Hour)}}} input, err := json.Marshal(jobs) if err != nil { t.Fatal(err) @@ -21,7 +21,7 @@ func TestScheduleDefinitionComputesFirstRunAndRejectsDuplicateIDs(t *testing.T) if err != nil { t.Fatal(err) } - var normalized []cronstore.JobSpec + var normalized []scheduling.JobSpec if err := json.Unmarshal(got, &normalized); err != nil { t.Fatal(err) } diff --git a/internal/app/controlplane/schema.go b/internal/app/controlplane/schema.go index da181275f..ca28599d2 100644 --- a/internal/app/controlplane/schema.go +++ b/internal/app/controlplane/schema.go @@ -17,7 +17,7 @@ const bareObjectSchema = `{"type":"object"}` var timeType = reflect.TypeFor[time.Time]() // stdDuration is the marker this tree's string-form duration types satisfy: -// config.Duration, channelDuration and cronstore.Duration each expose the +// config.Duration, channelDuration and scheduling.Duration each expose the // standard duration for arithmetic. var stdDuration = reflect.TypeOf((*interface{ Std() time.Duration })(nil)).Elem() @@ -28,7 +28,7 @@ var stdDuration = reflect.TypeOf((*interface{ Std() time.Duration })(nil)).Elem( // stale the moment a fourth duration type appears and the failure is silent in // the worst direction: the field derives as an integer and a client renders a // number box for a duration. That is exactly what the enumerated version of this -// function did to cronstore.Duration when the schedules interval became one. +// function did to scheduling.Duration when the schedules interval became one. func durationLike(t reflect.Type) bool { if !t.Implements(stdDuration) { return false diff --git a/internal/domain/scheduling/jobspec.go b/internal/domain/scheduling/jobspec.go new file mode 100644 index 000000000..1a5da4d52 --- /dev/null +++ b/internal/domain/scheduling/jobspec.go @@ -0,0 +1,112 @@ +package scheduling + +import ( + "errors" + "time" +) + +// Kind values for JobSpec.Kind: the discriminator that decides which delivery +// runner handles a job. +// +// They live beside the field that carries them because they are part of the +// schedules vocabulary an operator writes into the schedules document, a strict +// decoder reads back, and a build that never heard of a kind must reject loudly +// rather than deliver the wrong way. Which runner serves a kind is not this +// package's business: the router in internal/infrastructure/crondelivery owns +// that mapping, which is why an unrecognised kind is stored verbatim rather +// than refused at write time. +const ( + // KindChat delivers the job's Payload.Text as a chat message through the + // deployment's channel. It is the default: an empty Kind means KindChat, + // so every job written before this field existed keeps delivering the + // way it did when it was created. + KindChat = "chat" + + // KindWorkflow submits the job as a unit of work through the work-intake + // path, so the existing task lifecycle owns scheduling, retries, gates + // and PR opening. + KindWorkflow = "workflow" +) + +// JobSpec is one scheduled job as the schedules document persists it, plus +// the bookkeeping fields the scheduler computes (NextRun, LastRun, Created, +// Updated). The wire shape is the public API; the scheduling.Job returned +// to the engine is a stripped subset. +type JobSpec struct { + // ID is the unique identifier. Required, non-empty, and the merge + // key for run recording. + ID string `json:"id"` + + // Pool is "parallel" or "sequential". Empty means sequential — the + // safer default, matching Pool.resolve(). + Pool string `json:"pool,omitempty"` + + // Detail is a human-readable label rendered on events. + Detail string `json:"detail,omitempty"` + + // Kind selects which delivery runner handles this job. Empty means + // KindChat, so a job written before this field existed keeps + // delivering as a chat message. This package does not interpret the + // value: an unrecognised kind is stored verbatim and refused by the + // delivery router, which is the one place the mapping from kind to + // runner lives. + Kind string `json:"kind,omitempty"` + + // Schedule decides when this job is due. See schedule.go for the + // supported kinds and their arithmetic. + Schedule Schedule `json:"schedule"` + + // Target and Payload describe where the job's output goes and what it + // carries. They live on the wire so jobs created ahead of the delivery + // runners that read them do not need a shape migration. + Target Target `json:"target"` + Payload Payload `json:"payload"` + + // NextRun is the absolute moment the job will next fire. Computed + // at write time so the engine's hot path is a single comparison. + NextRun time.Time `json:"next_run"` + + // LastRun is the last run time reported by the run-recording step, + // nil if the job has never run. The schedule does not know whether + // the run succeeded — that judgement belongs to the runner. + LastRun *time.Time `json:"last_run,omitempty"` + + // Created and Updated are audit metadata written by the document's + // owner; the engine does not read them. + Created time.Time `json:"created"` + Updated time.Time `json:"updated"` +} + +// Target identifies where the job's output goes. Empty Target is valid — +// the delivery runner chooses a default. +type Target struct { + // ChatID is the chat channel id (Telegram chat id, email address, + // webhook URL — depends on the channel). Empty defers to a default + // chosen by the delivery runner. + ChatID string `json:"chat_id,omitempty"` +} + +// Payload is the content sent at run time. +type Payload struct { + // Text is the literal message body. Templating is a future concern + // and is deliberately absent here. + Text string `json:"text,omitempty"` +} + +// Sentinels the schedule vocabulary and its callers match with errors.Is. +var ( + // ErrJobNotFound is returned by a job source when the requested id + // is absent from its document. + ErrJobNotFound = errors.New("scheduling: job not found") + + // ErrInvalidSpec is returned when a JobSpec or Schedule fails + // structural validation (empty id, unknown schedule kind, + // non-positive interval). + ErrInvalidSpec = errors.New("scheduling: invalid job spec") + + // ErrScheduleUnsupported is returned when a job's schedule kind + // cannot compute its next run time. Interval arithmetic is fully + // implemented; cron and once return this so callers can surface a + // clear error rather than silently advancing. + ErrScheduleUnsupported = errors.New("scheduling: schedule kind not implemented") +) diff --git a/internal/infrastructure/cronstore/schedule.go b/internal/domain/scheduling/schedule.go similarity index 99% rename from internal/infrastructure/cronstore/schedule.go rename to internal/domain/scheduling/schedule.go index f08ecfadb..25d044645 100644 --- a/internal/infrastructure/cronstore/schedule.go +++ b/internal/domain/scheduling/schedule.go @@ -1,4 +1,4 @@ -package cronstore +package scheduling import ( "encoding/json" diff --git a/internal/infrastructure/crondelivery/contract.go b/internal/infrastructure/crondelivery/contract.go index 0496c772b..a65b2dc31 100644 --- a/internal/infrastructure/crondelivery/contract.go +++ b/internal/infrastructure/crondelivery/contract.go @@ -29,24 +29,23 @@ import ( "time" "github.com/samcharles93/archie-core/internal/domain/scheduling" - "github.com/samcharles93/archie-core/internal/infrastructure/cronstore" ) // SpecLookup resolves a job id to its persisted spec. // -// It is declared here (consumer-owned) rather than imported from cronstore as -// a concrete type so this package depends on the one method it uses. -// *cronstore.Store satisfies it as written, so a deployment passes the store -// straight in. +// It is declared here (consumer-owned) rather than imported from the document's +// owner as a concrete type so this package depends on the one method it uses. +// scheduleResourceStore in internal/app/archied satisfies it as written, so a +// deployment passes that in. type SpecLookup interface { // Get returns the job and whether it was found; the error is reserved - // for I/O failures, matching cronstore.Store.Get. - Get(ctx context.Context, id string) (cronstore.JobSpec, bool, error) + // for I/O failures, matching scheduleResourceStore.Get. + Get(ctx context.Context, id string) (scheduling.JobSpec, bool, error) } // RunRecorder advances a job's schedule once a run has completed. It is // declared here (consumer-owned) for the same reason SpecLookup is, and -// *cronstore.Store satisfies it as written. +// scheduleResourceStore satisfies it as written. // // MarkRun is the only step that moves a job's NextRun, so a store whose Due // keeps reporting an already-completed job keeps handing it back on every @@ -98,20 +97,18 @@ var errSpecMissing = errors.New("crondelivery: job spec not found") // hydrate loads the job's persisted spec, wrapping both a store error and the // not-found case so a caller can match on one sentinel. -func hydrate(ctx context.Context, specs SpecLookup, job scheduling.Job) (cronstore.JobSpec, error) { +func hydrate(ctx context.Context, specs SpecLookup, job scheduling.Job) (scheduling.JobSpec, error) { spec, ok, err := specs.Get(ctx, job.ID) if err != nil { - return cronstore.JobSpec{}, fmt.Errorf("crondelivery: load job %q: %w", job.ID, err) + return scheduling.JobSpec{}, fmt.Errorf("crondelivery: load job %q: %w", job.ID, err) } if !ok { - return cronstore.JobSpec{}, fmt.Errorf("%w: id %q", errSpecMissing, job.ID) + return scheduling.JobSpec{}, fmt.Errorf("%w: id %q", errSpecMissing, job.ID) } return spec, nil } -// compile-time check: the store is the lookup a deployment hands in unchanged. -var ( - _ SpecLookup = (*cronstore.Store)(nil) - _ RunRecorder = (*cronstore.Store)(nil) - _ RouterStore = (*cronstore.Store)(nil) -) +// Compile-time check that the production store satisfies the contract. It +// lives in internal/app/archied, which imports this package, so the assertion +// is made there (scheduling.go) rather than here, where it would be an import +// cycle. diff --git a/internal/infrastructure/crondelivery/helpers_test.go b/internal/infrastructure/crondelivery/helpers_test.go index b162db762..d12791229 100644 --- a/internal/infrastructure/crondelivery/helpers_test.go +++ b/internal/infrastructure/crondelivery/helpers_test.go @@ -6,13 +6,12 @@ package crondelivery import ( "context" - "path/filepath" + "fmt" "sync" "testing" "time" "github.com/samcharles93/archie-core/internal/domain/scheduling" - "github.com/samcharles93/archie-core/internal/infrastructure/cronstore" ) // --- fakes -------------------------------------------------------------------- @@ -88,24 +87,24 @@ func (n *naiveRunner) count() int { // fakeLookup serves specs from a map and ignores ctx entirely, like // naiveCourier. // -// The cancellation tests need it because the real store is itself -// cancellation-aware: *cronstore.Store.Get returns ctx.Err() before it reads, -// so with a real store the runner's own guard is masked — hydrate fails and the -// run returns the context error whether or not the runner checked. A lookup -// that ignores ctx removes that mask, leaving the runner's guard as the only -// thing that can prevent the side effect. +// The cancellation tests need it because a production store is expected to be +// cancellation-aware: a source whose Get returns ctx.Err() before it reads +// masks the runner's own guard — hydrate fails and the run returns the context +// error whether or not the runner checked. A lookup that ignores ctx removes +// that mask, leaving the runner's guard as the only thing that can prevent the +// side effect. type fakeLookup struct { - specs map[string]cronstore.JobSpec + specs map[string]scheduling.JobSpec } -func (f *fakeLookup) Get(_ context.Context, id string) (cronstore.JobSpec, bool, error) { +func (f *fakeLookup) Get(_ context.Context, id string) (scheduling.JobSpec, bool, error) { spec, ok := f.specs[id] return spec, ok, nil } // MarkRun satisfies RouterStore. It deliberately does nothing: these tests // exercise the runner's and the router's own dispatch guards, and the record -// step is covered against the real store in +// step is covered against a store that really advances in // TestRouterAdvancesTheScheduleAfterASuccessfulRun. func (f *fakeLookup) MarkRun(_ context.Context, _ string, _ time.Time) error { return nil } @@ -247,20 +246,80 @@ func (c *countingRunner) count() int { // --- helpers ------------------------------------------------------------------ -// newStore opens a real cronstore on a temp path and closes it on cleanup. -func newStore(t *testing.T) *cronstore.Store { - t.Helper() - s, err := cronstore.Open(filepath.Join(t.TempDir(), "jobs.json")) - if err != nil { - t.Fatalf("cronstore.Open: %v", err) - } - t.Cleanup(func() { _ = s.Close() }) - return s +// memStore is the test's schedules document: a map with the three faces the +// production store shows the package — SpecLookup (Get), RunRecorder (MarkRun +// with real schedule arithmetic), and JobSource (Due). It replaces the file- +// backed store this package's fixtures used to open, which died with the +// legacy jobs.json store; the document now lives behind the control plane in +// production, and what these tests need from persistence is semantics, not a +// file. +type memStore struct { + mu sync.Mutex + jobs map[string]scheduling.JobSpec } -// createJob persists a spec through the real store and returns the scheduling -// job the engine would hand a runner for it. -func createJob(t *testing.T, s *cronstore.Store, spec cronstore.JobSpec) scheduling.Job { +func newStore(t *testing.T) *memStore { + t.Helper() + return &memStore{jobs: make(map[string]scheduling.JobSpec)} +} + +// Create persists a spec verbatim. Fixtures that want a due job set NextRun +// themselves (createJob does it when unset). +func (m *memStore) Create(_ context.Context, spec scheduling.JobSpec) error { + m.mu.Lock() + defer m.mu.Unlock() + if _, exists := m.jobs[spec.ID]; exists { + return fmt.Errorf("memStore: id %q already exists", spec.ID) + } + m.jobs[spec.ID] = spec + return nil +} + +func (m *memStore) Get(_ context.Context, id string) (scheduling.JobSpec, bool, error) { + m.mu.Lock() + defer m.mu.Unlock() + spec, ok := m.jobs[id] + return spec, ok, nil +} + +// Due returns the jobs whose NextRun is at or before now, the same contract +// the production source keeps. +func (m *memStore) Due(_ context.Context, now time.Time) ([]scheduling.Job, error) { + m.mu.Lock() + defer m.mu.Unlock() + due := make([]scheduling.Job, 0, len(m.jobs)) + for _, spec := range m.jobs { + if !spec.NextRun.After(now) { + due = append(due, scheduling.Job{ID: spec.ID, Pool: scheduling.Pool(spec.Pool), Detail: spec.Detail}) + } + } + return due, nil +} + +// MarkRun advances an interval job's NextRun the way the production +// recording step does, and reports a once schedule as unsupported — the +// sentinel the router swallows. +func (m *memStore) MarkRun(_ context.Context, id string, runAt time.Time) error { + m.mu.Lock() + defer m.mu.Unlock() + spec, ok := m.jobs[id] + if !ok { + return fmt.Errorf("%w: id %q", scheduling.ErrJobNotFound, id) + } + next, err := spec.Schedule.NextRun(runAt) + if err != nil { + return err + } + at := runAt.UTC() + spec.LastRun = &at + spec.NextRun = next + m.jobs[id] = spec + return nil +} + +// createJob persists a spec and returns the scheduling job the engine would +// hand a runner for it. +func createJob(t *testing.T, s *memStore, spec scheduling.JobSpec) scheduling.Job { t.Helper() if spec.NextRun.IsZero() { // Due now: the engine only ever hands a runner a job its source @@ -268,30 +327,30 @@ func createJob(t *testing.T, s *cronstore.Store, spec cronstore.JobSpec) schedul spec.NextRun = time.Now().UTC().Add(-time.Minute) } if err := s.Create(t.Context(), spec); err != nil { - t.Fatalf("cronstore.Create(%q): %v", spec.ID, err) + t.Fatalf("Create(%q): %v", spec.ID, err) } return scheduling.Job{ID: spec.ID, Pool: scheduling.Pool(spec.Pool), Detail: spec.Detail} } // chatSpec is a minimal chat job. -func chatSpec(id, chatID, text string) cronstore.JobSpec { - return cronstore.JobSpec{ +func chatSpec(id, chatID, text string) scheduling.JobSpec { + return scheduling.JobSpec{ ID: id, Detail: "test job " + id, - Kind: cronstore.KindChat, - Schedule: cronstore.Schedule{Kind: cronstore.ScheduleInterval, Interval: cronstore.Duration(time.Hour)}, - Target: cronstore.Target{ChatID: chatID}, - Payload: cronstore.Payload{Text: text}, + Kind: scheduling.KindChat, + Schedule: scheduling.Schedule{Kind: scheduling.ScheduleInterval, Interval: scheduling.Duration(time.Hour)}, + Target: scheduling.Target{ChatID: chatID}, + Payload: scheduling.Payload{Text: text}, } } // workflowSpec is a minimal workflow job. -func workflowSpec(id, detail, body string) cronstore.JobSpec { - return cronstore.JobSpec{ +func workflowSpec(id, detail, body string) scheduling.JobSpec { + return scheduling.JobSpec{ ID: id, Detail: detail, - Kind: cronstore.KindWorkflow, - Schedule: cronstore.Schedule{Kind: cronstore.ScheduleInterval, Interval: cronstore.Duration(time.Hour)}, - Payload: cronstore.Payload{Text: body}, + Kind: scheduling.KindWorkflow, + Schedule: scheduling.Schedule{Kind: scheduling.ScheduleInterval, Interval: scheduling.Duration(time.Hour)}, + Payload: scheduling.Payload{Text: body}, } } diff --git a/internal/infrastructure/crondelivery/router.go b/internal/infrastructure/crondelivery/router.go index d2421e8a3..fc73da673 100644 --- a/internal/infrastructure/crondelivery/router.go +++ b/internal/infrastructure/crondelivery/router.go @@ -9,7 +9,6 @@ import ( "github.com/samcharles93/archie-core/internal/domain/scheduling" "github.com/samcharles93/archie-core/internal/events" - "github.com/samcharles93/archie-core/internal/infrastructure/cronstore" ) // phaseDispatch names the step that failed on a KindJobError the Router emits. @@ -43,7 +42,7 @@ type Router struct { // emission — the router still refuses, it is just unobservable, the same // tolerance the engine has for its own nil sink. // -// An empty kind resolves to cronstore.KindChat, so a deployment only needs a +// An empty kind resolves to scheduling.KindChat, so a deployment only needs a // mapping for the kinds it actually uses. func NewRouter(specs RouterStore, runners map[string]scheduling.Runner, sink scheduling.Sink) (*Router, error) { if specs == nil { @@ -67,7 +66,7 @@ func (r *Router) Run(ctx context.Context, job scheduling.Job) error { kind := spec.Kind if kind == "" { - kind = cronstore.KindChat + kind = scheduling.KindChat } runner, ok := r.runners[kind] if !ok { @@ -86,12 +85,12 @@ func (r *Router) Run(ctx context.Context, job scheduling.Job) error { // per tick instead. // // A schedule kind with no recurring next run -- a one-shot -- reports -// cronstore.ErrScheduleUnsupported, which is the job's definition rather than a +// scheduling.ErrScheduleUnsupported, which is the job's definition rather than a // run failure, so it is swallowed. Anything else is returned: the run happened // but the bookkeeping did not, and hiding that would leave the job silently // re-firing. func (r *Router) recordRun(ctx context.Context, job scheduling.Job) error { - if err := r.specs.MarkRun(ctx, job.ID, time.Now()); err != nil && !errors.Is(err, cronstore.ErrScheduleUnsupported) { + if err := r.specs.MarkRun(ctx, job.ID, time.Now()); err != nil && !errors.Is(err, scheduling.ErrScheduleUnsupported) { return fmt.Errorf("crondelivery: record run of job %q: %w", job.ID, err) } return nil diff --git a/internal/infrastructure/crondelivery/router_test.go b/internal/infrastructure/crondelivery/router_test.go index 8f85cf4d4..63d557c6d 100644 --- a/internal/infrastructure/crondelivery/router_test.go +++ b/internal/infrastructure/crondelivery/router_test.go @@ -8,7 +8,6 @@ import ( "github.com/samcharles93/archie-core/internal/domain/scheduling" "github.com/samcharles93/archie-core/internal/events" - "github.com/samcharles93/archie-core/internal/infrastructure/cronstore" ) // --- Router ------------------------------------------------------------------- @@ -19,17 +18,17 @@ import ( func routerFixture(t *testing.T, kind string) (*Router, *countingRunner, *countingRunner, *recordingSink) { t.Helper() s := newStore(t) - createJob(t, s, cronstore.JobSpec{ + createJob(t, s, scheduling.JobSpec{ ID: "routed", Detail: "routed job", Kind: kind, - Schedule: cronstore.Schedule{Kind: cronstore.ScheduleInterval, Interval: cronstore.Duration(time.Hour)}, + Schedule: scheduling.Schedule{Kind: scheduling.ScheduleInterval, Interval: scheduling.Duration(time.Hour)}, }) chat, workflow := &countingRunner{}, &countingRunner{} sink := &recordingSink{} r, err := NewRouter(s, map[string]scheduling.Runner{ - cronstore.KindChat: chat, - cronstore.KindWorkflow: workflow, + scheduling.KindChat: chat, + scheduling.KindWorkflow: workflow, }, sink) if err != nil { t.Fatalf("NewRouter: %v", err) @@ -44,8 +43,8 @@ func TestRouterDispatchesByKind(t *testing.T) { wantChat int wantWorkflow int }{ - {"chat kind reaches the chat runner", cronstore.KindChat, 1, 0}, - {"workflow kind reaches the workflow runner", cronstore.KindWorkflow, 0, 1}, + {"chat kind reaches the chat runner", scheduling.KindChat, 1, 0}, + {"workflow kind reaches the workflow runner", scheduling.KindWorkflow, 0, 1}, {"empty kind defaults to the chat runner", "", 1, 0}, } for _, tc := range tests { @@ -117,15 +116,15 @@ func TestRouterUnknownKindIsRefusedAtTheGate(t *testing.T) { // tolerance: a nil sink disables emission without disabling the refusal. func TestRouterNilSinkDoesNotPanicOnUnknownKind(t *testing.T) { s := newStore(t) - createJob(t, s, cronstore.JobSpec{ + createJob(t, s, scheduling.JobSpec{ ID: "routed", Kind: "teleport", - Schedule: cronstore.Schedule{Kind: cronstore.ScheduleInterval, Interval: cronstore.Duration(time.Hour)}, + Schedule: scheduling.Schedule{Kind: scheduling.ScheduleInterval, Interval: scheduling.Duration(time.Hour)}, }) chat, workflow := &countingRunner{}, &countingRunner{} r, err := NewRouter(s, map[string]scheduling.Runner{ - cronstore.KindChat: chat, - cronstore.KindWorkflow: workflow, + scheduling.KindChat: chat, + scheduling.KindWorkflow: workflow, }, nil) if err != nil { t.Fatalf("NewRouter: %v", err) @@ -143,7 +142,7 @@ func TestRouterNilSinkDoesNotPanicOnUnknownKind(t *testing.T) { func TestRouterMissingSpecReturnsError(t *testing.T) { s := newStore(t) r, err := NewRouter(s, map[string]scheduling.Runner{ - cronstore.KindChat: &countingRunner{}, + scheduling.KindChat: &countingRunner{}, }, &recordingSink{}) if err != nil { t.Fatalf("NewRouter: %v", err) @@ -161,7 +160,7 @@ func TestRouterMissingSpecReturnsError(t *testing.T) { // the job the engine built, not a re-derived one: identity and pool are the // engine's, and the runner's event attribution depends on them surviving. func TestRouterPassesTheEngineJobThrough(t *testing.T) { - r, chat, _, _ := routerFixture(t, cronstore.KindChat) + r, chat, _, _ := routerFixture(t, scheduling.KindChat) job := scheduling.Job{ID: "routed", Pool: scheduling.PoolSequential, Detail: "detail from engine"} if err := r.Run(t.Context(), job); err != nil { @@ -185,7 +184,7 @@ func TestRouterPassesTheEngineJobThrough(t *testing.T) { // honours cancellation would make this pass whether or not the runner checks, // which is the failure mode the criterion exists to prevent. func TestEveryRunnerHonoursCancellation(t *testing.T) { - lookup := &fakeLookup{specs: map[string]cronstore.JobSpec{ + lookup := &fakeLookup{specs: map[string]scheduling.JobSpec{ "chat-job": chatSpec("chat-job", "ops-room", "hello"), "wf-job": workflowSpec("wf-job", "workflow job", "body"), }} @@ -204,8 +203,8 @@ func TestEveryRunnerHonoursCancellation(t *testing.T) { // router's own guard: a cancelled run must not reach the dispatch. forwarded := &naiveRunner{} router, err := NewRouter(lookup, map[string]scheduling.Runner{ - cronstore.KindChat: forwarded, - cronstore.KindWorkflow: workflow, + scheduling.KindChat: forwarded, + scheduling.KindWorkflow: workflow, }, &recordingSink{}) if err != nil { t.Fatalf("NewRouter: %v", err) @@ -246,27 +245,27 @@ func TestEveryRunnerHonoursCancellation(t *testing.T) { const dailyStatusSummary = "Daily status summary: all workers healthy, no tasks parked." // TestDailyStatusSummaryEndToEnd is the epic's headline scenario exercised -// through the whole chain a deployment uses: a real job persisted in a real -// cronstore is reported due by the store's own JobSource, handed to the router +// through the whole chain a deployment uses: a scheduled job in the schedules +// document is reported due by its own JobSource, handed to the router // the engine is given, dispatched to the chat runner, and delivered as the // composed message. func TestDailyStatusSummaryEndToEnd(t *testing.T) { s := newStore(t) const chatID = "ops-room" - spec := cronstore.JobSpec{ + spec := scheduling.JobSpec{ ID: "daily-status", Detail: "daily status summary", - Kind: cronstore.KindChat, - Schedule: cronstore.Schedule{Kind: cronstore.ScheduleInterval, Interval: cronstore.Duration(24 * time.Hour)}, - Target: cronstore.Target{ChatID: chatID}, - Payload: cronstore.Payload{Text: dailyStatusSummary}, + Kind: scheduling.KindChat, + Schedule: scheduling.Schedule{Kind: scheduling.ScheduleInterval, Interval: scheduling.Duration(24 * time.Hour)}, + Target: scheduling.Target{ChatID: chatID}, + Payload: scheduling.Payload{Text: dailyStatusSummary}, // Due now: create it with a next_run in the past, the state a // long-idle daemon wakes up to. NextRun: time.Now().UTC().Add(-time.Minute), } if err := s.Create(t.Context(), spec); err != nil { - t.Fatalf("cronstore.Create: %v", err) + t.Fatalf("Create: %v", err) } // The engine's source: the store itself, answering what is due. @@ -284,7 +283,7 @@ func TestDailyStatusSummaryEndToEnd(t *testing.T) { t.Fatalf("NewChatCourier: %v", err) } router, err := NewRouter(s, map[string]scheduling.Runner{ - cronstore.KindChat: courierRunner, + scheduling.KindChat: courierRunner, }, &recordingSink{}) if err != nil { t.Fatalf("NewRouter: %v", err) @@ -334,7 +333,7 @@ func TestConstructorsRejectNilDependencies(t *testing.T) { }}, {"Router without a spec lookup", func() error { _, err := NewRouter(nil, map[string]scheduling.Runner{ - cronstore.KindChat: &countingRunner{}, + scheduling.KindChat: &countingRunner{}, }, nil) return err }}, @@ -363,7 +362,7 @@ func TestImplementsSchedulingRunner(t *testing.T) { } // TestRouterAdvancesTheScheduleAfterASuccessfulRun is the acceptance criterion -// for a recurring job's timing: cronstore.Store.MarkRun is the only step that +// for a recurring job's timing: the recording step is the only thing that // moves a job's NextRun, and the Router is the single Runner the engine hands // every job. With no caller the store keeps reporting the job due on every // tick, so a job scheduled every hour fires once per tick instead. @@ -380,7 +379,7 @@ func TestRouterAdvancesTheScheduleAfterASuccessfulRun(t *testing.T) { } r, err := NewRouter(s, map[string]scheduling.Runner{ - cronstore.KindChat: &countingRunner{}, + scheduling.KindChat: &countingRunner{}, }, &recordingSink{}) if err != nil { t.Fatalf("NewRouter: %v", err) @@ -404,23 +403,23 @@ func TestRouterAdvancesTheScheduleAfterASuccessfulRun(t *testing.T) { // TestRouterToleratesAOnceScheduleWithoutANextRun pins the other half: a // one-shot job has no recurring next run by definition, so MarkRun reports -// cronstore.ErrScheduleUnsupported. That is not a run failure -- the job ran -- +// scheduling.ErrScheduleUnsupported. That is not a run failure -- the job ran -- // and returning it would make the engine emit KindJobError for a job that // succeeded. func TestRouterToleratesAOnceScheduleWithoutANextRun(t *testing.T) { s := newStore(t) at := time.Now().Add(-time.Hour) - job := createJob(t, s, cronstore.JobSpec{ + job := createJob(t, s, scheduling.JobSpec{ ID: "one-shot", Detail: "one-shot job", - Kind: cronstore.KindChat, - Schedule: cronstore.Schedule{Kind: cronstore.ScheduleOnce, At: &at}, - Target: cronstore.Target{ChatID: "chat-1"}, - Payload: cronstore.Payload{Text: "once"}, + Kind: scheduling.KindChat, + Schedule: scheduling.Schedule{Kind: scheduling.ScheduleOnce, At: &at}, + Target: scheduling.Target{ChatID: "chat-1"}, + Payload: scheduling.Payload{Text: "once"}, }) r, err := NewRouter(s, map[string]scheduling.Runner{ - cronstore.KindChat: &countingRunner{}, + scheduling.KindChat: &countingRunner{}, }, &recordingSink{}) if err != nil { t.Fatalf("NewRouter: %v", err) diff --git a/internal/infrastructure/crondelivery/runner_test.go b/internal/infrastructure/crondelivery/runner_test.go index 5701aac02..d7f559754 100644 --- a/internal/infrastructure/crondelivery/runner_test.go +++ b/internal/infrastructure/crondelivery/runner_test.go @@ -7,7 +7,6 @@ import ( "time" "github.com/samcharles93/archie-core/internal/domain/scheduling" - "github.com/samcharles93/archie-core/internal/infrastructure/cronstore" ) // --- ChatCourier -------------------------------------------------------------- @@ -83,7 +82,7 @@ func TestChatCourierHonoursCancellation(t *testing.T) { t.Run("cancelled before the run sends nothing", func(t *testing.T) { // Both the lookup and the courier ignore ctx, so only the // runner's own guard can stop the send. - lookup := &fakeLookup{specs: map[string]cronstore.JobSpec{ + lookup := &fakeLookup{specs: map[string]scheduling.JobSpec{ "cancel-me": chatSpec("cancel-me", "ops-room", "morning"), }} courier := &naiveCourier{} @@ -224,7 +223,7 @@ func TestWorkflowTaskHonoursCancellation(t *testing.T) { t.Run("cancelled before the run submits nothing", func(t *testing.T) { // Both the lookup and the submitter ignore ctx, so only the // runner's own guard can stop the submit. - lookup := &fakeLookup{specs: map[string]cronstore.JobSpec{ + lookup := &fakeLookup{specs: map[string]scheduling.JobSpec{ "cancel-me": workflowSpec("cancel-me", "cancel me", "body"), }} sub := &naiveSubmitter{} diff --git a/internal/infrastructure/cronstore/doc.go b/internal/infrastructure/cronstore/doc.go deleted file mode 100644 index d886acc7b..000000000 --- a/internal/infrastructure/cronstore/doc.go +++ /dev/null @@ -1,41 +0,0 @@ -// Package cronstore is the file-backed job store that backs the scheduling -// ticker engine. It is infrastructure: it owns the on-disk layout, the -// cross-process lock, and the version-stamped JSON schema. It does not own -// timing semantics — those live in internal/domain/scheduling. -// -// Two surfaces: -// -// - scheduling.JobSource: Due(ctx, now) — what the engine consumes. -// - CRUD on JobSpec — what the CLI and dashboard consume. -// -// Both share the same Store value, the same lock, and the same on-disk -// file. The engine applies no schedule arithmetic of its own; whatever -// "due" means is the store's decision, including future kinds the store -// grows into. -// -// Concurrency model: every read and write acquires an OS-level flock on a -// sibling lock file, so a CLI edit and a daemon writer cannot tear the file. -// Conflict resolution is last-writer-wins: the second writer overwrites the -// first's whole document. The lock makes that serial rather than racy. -// In-process, two Store values for the same path are serialised by a -// package-level mutex keyed by lock-file path — flock is per file -// description on Linux and would not block them otherwise. -// -// File format (versioned; bump schemaVersion on ANY shape change -// including additive ones, since DisallowUnknownFields rejects them): -// -// { -// "schema_version": 2, -// "jobs": [ JobSpec, ... ] -// } -// -// Version 2 added JobSpec.Kind, the delivery discriminator. A version-1 -// file still loads — Kind reads back empty, which the delivery router -// resolves to the chat default — so jobs created before the field existed -// keep firing the way they always did. -// -// JobSpec carries the engine-facing fields it always needed (ID, Pool, -// Detail) plus everything the store computes for it (NextRun, LastRun, -// Created, Updated). Merge semantics on Update mean a nil field in a Patch -// leaves the corresponding persisted field untouched. -package cronstore diff --git a/internal/infrastructure/cronstore/errors.go b/internal/infrastructure/cronstore/errors.go deleted file mode 100644 index 2bd7268db..000000000 --- a/internal/infrastructure/cronstore/errors.go +++ /dev/null @@ -1,39 +0,0 @@ -package cronstore - -import "errors" - -// Sentinel errors callers can match with errors.Is. -var ( - // ErrUnsupportedSchema is returned by Open when the persisted file - // declares a schema version this build does not understand. The - // store treats that as an explicit refusal to read: a forward - // version may have shape we cannot safely render back out, and - // silently downgrading would lose data. - ErrUnsupportedSchema = errors.New("cronstore: unsupported schema version") - - // ErrCorruptFile is returned by Open when the file exists but does - // not decode as a valid envelope: truncated, garbage, or fields of - // the wrong shape. The store never panics on bad input, so the - // engine can surface this as phase="source" KindJobError. - ErrCorruptFile = errors.New("cronstore: corrupt file") - - // ErrJobExists is returned by Create when the requested id is - // already present. Callers that want upsert semantics should - // follow up with Update. - ErrJobExists = errors.New("cronstore: job already exists") - - // ErrJobNotFound is returned by Get, Update and MarkRun when the - // id is absent. Delete is the exception: it is idempotent. - ErrJobNotFound = errors.New("cronstore: job not found") - - // ErrInvalidSpec is returned by Create and Update when the supplied - // JobSpec / Patch fails the store's structural validation (empty id, - // unknown schedule kind, non-positive interval). - ErrInvalidSpec = errors.New("cronstore: invalid job spec") - - // ErrScheduleUnsupported is returned by MarkRun when the job's - // schedule kind cannot compute its next run time. Slice 1 only - // knows interval arithmetic; cron / once return this so callers - // can surface a clear error rather than silently advancing. - ErrScheduleUnsupported = errors.New("cronstore: schedule kind not implemented") -) diff --git a/internal/infrastructure/cronstore/kind.go b/internal/infrastructure/cronstore/kind.go deleted file mode 100644 index 128a2e7cd..000000000 --- a/internal/infrastructure/cronstore/kind.go +++ /dev/null @@ -1,24 +0,0 @@ -package cronstore - -// Kind values for JobSpec.Kind: the discriminator that decides which delivery -// runner handles a job. -// -// They live here, beside the field that carries them, because they are part of -// the on-disk vocabulary — an operator writes one in jobs.json, the strict -// decoder reads it back, and a file written by a build that never heard of it -// must fail loudly rather than deliver the wrong way. Which runner serves a -// kind is not the store's business: the router in -// internal/infrastructure/crondelivery owns that mapping, which is why an -// unrecognised kind is left on disk rather than refused at Create. -const ( - // KindChat delivers the job's Payload.Text as a chat message through the - // deployment's channel. It is the default: an empty Kind means KindChat, - // so every job written before this field existed (schema version 1) - // keeps delivering the way it did when it was created. - KindChat = "chat" - - // KindWorkflow submits the job as a unit of work through the work-intake - // path, so the existing task lifecycle owns scheduling, retries, gates - // and PR opening. - KindWorkflow = "workflow" -) diff --git a/internal/infrastructure/cronstore/kind_test.go b/internal/infrastructure/cronstore/kind_test.go deleted file mode 100644 index e4e4d8d06..000000000 --- a/internal/infrastructure/cronstore/kind_test.go +++ /dev/null @@ -1,148 +0,0 @@ -// Tests for the JobSpec.Kind discriminator added in the cron-delivery slice. -// -// The store owns the persisted vocabulary; the runner mapping belongs to -// internal/infrastructure/crondelivery. These tests pin the store-side -// properties that mapping depends on: the field round-trips, an empty value -// reads back as the chat default, and a file written before the field existed -// still loads. -package cronstore - -import ( - "bytes" - "context" - "os" - "path/filepath" - "testing" - "time" -) - -// TestPersistedFileStampsCurrentSchemaVersion guards the additive-change rule -// documented on schemaVersion: adding Kind to JobSpec forces a version bump, -// because DisallowUnknownFields means an older build must reject a file -// carrying a field it does not know rather than silently dropping it. -func TestPersistedFileStampsCurrentSchemaVersion(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "jobs.json") - s, err := Open(path) - if err != nil { - t.Fatalf("Open: %v", err) - } - defer s.Close() - mustCreate(t, s, fixedIntervalSpec("x", time.Minute)) - if err := s.Close(); err != nil { - t.Fatalf("Close: %v", err) - } - - data, err := os.ReadFile(path) - if err != nil { - t.Fatalf("ReadFile: %v", err) - } - if !bytes.Contains(data, []byte(`"schema_version":2`)) { - t.Errorf("file does not carry the bumped schema_version:2 stamp: %s", data) - } -} - -// TestKindRoundTrips pins that the discriminator survives a write and read on -// the real file, not just in memory: Create serialises it, a fresh Open with -// the strict decoder must read it back. -func TestKindRoundTrips(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "jobs.json") - - s, err := Open(path) - if err != nil { - t.Fatalf("Open: %v", err) - } - spec := fixedIntervalSpec("nightly-build", time.Hour) - spec.Kind = KindWorkflow - mustCreate(t, s, spec) - if err := s.Close(); err != nil { - t.Fatalf("Close: %v", err) - } - - // A second handle proves the value is on disk and survives the strict - // decoder, rather than only living in the writer's in-memory mirror. - reopened, err := Open(path) - if err != nil { - t.Fatalf("reopen: %v", err) - } - defer reopened.Close() - - got, ok, err := reopened.Get(context.Background(), "nightly-build") - if err != nil { - t.Fatalf("Get: %v", err) - } - if !ok { - t.Fatal("Get returned ok=false after reopen") - } - if got.Kind != KindWorkflow { - t.Errorf("Kind = %q, want %q", got.Kind, KindWorkflow) - } -} - -// TestEmptyKindLoadsAsChatDefault pins the compatibility rule the router -// relies on: a job that names no kind delivers as a chat message. Asserting -// the empty string (rather than KindChat) is deliberate — the store must not -// rewrite what the operator wrote; resolving empty to "chat" is the router's -// job, and this test pins that the store hands it through untouched. -func TestEmptyKindLoadsAsChatDefault(t *testing.T) { - s := newTestStore(t) - mustCreate(t, s, fixedIntervalSpec("no-kind", time.Minute)) - - got, ok, err := s.Get(context.Background(), "no-kind") - if err != nil { - t.Fatalf("Get: %v", err) - } - if !ok { - t.Fatal("Get returned ok=false") - } - if got.Kind != "" { - t.Errorf("Kind = %q, want empty (the chat default)", got.Kind) - } -} - -// TestVersion1FileWithoutKindStillLoads is the migration guard for the schema -// bump. A file written by Slice 1 has schema_version 1 and no kind field; the -// bumped build must load it rather than rejecting it, or every existing cron -// job stops firing the moment this ships. -func TestVersion1FileWithoutKindStillLoads(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "jobs.json") - past := time.Now().Add(-time.Minute).UTC().Format(time.RFC3339Nano) - contents := `{"schema_version":1,"jobs":[{"id":"legacy","pool":"sequential",` + - `"detail":"written before kind existed",` + - `"schedule":{"kind":"interval","interval":60000000000},` + - `"target":{"chat_id":"ops"},"payload":{"text":"legacy payload"},` + - `"next_run":"` + past + `","created":"` + past + `","updated":"` + past + `"}]}` - if err := os.WriteFile(path, []byte(contents), 0o600); err != nil { - t.Fatalf("WriteFile: %v", err) - } - - s, err := Open(path) - if err != nil { - t.Fatalf("Open on a version-1 file after the bump = %v, want nil", err) - } - defer s.Close() - - got, ok, err := s.Get(context.Background(), "legacy") - if err != nil { - t.Fatalf("Get: %v", err) - } - if !ok { - t.Fatal("Get returned ok=false for a version-1 job") - } - if got.Kind != "" { - t.Errorf("Kind = %q, want empty for a version-1 job", got.Kind) - } - if got.Payload.Text != "legacy payload" { - t.Errorf("Payload.Text = %q, want %q", got.Payload.Text, "legacy payload") - } - // It must still be dispatchable: the engine reads due jobs through Due. - due, err := s.Due(context.Background(), time.Now()) - if err != nil { - t.Fatalf("Due: %v", err) - } - if len(due) != 1 || due[0].ID != "legacy" { - t.Errorf("Due on a version-1 file = %+v, want the legacy job", due) - } -} diff --git a/internal/infrastructure/cronstore/lock_unix.go b/internal/infrastructure/cronstore/lock_unix.go deleted file mode 100644 index d2734da9d..000000000 --- a/internal/infrastructure/cronstore/lock_unix.go +++ /dev/null @@ -1,68 +0,0 @@ -//go:build unix - -package cronstore - -import ( - "fmt" - "os" - "syscall" -) - -// lockedFile is an OS-level flock wrapper around a sidecar file. The -// store acquires it before reading or writing the JSON, so a CLI edit -// and a daemon writer cannot tear the same jobs.json. -// -// flock is per-open-file-description, which means a single Store holds -// the lock for as long as it lives; concurrent processes opening the -// same sidecar queue behind it. -type lockedFile struct { - f *os.File -} - -// openLock creates (or opens) the lock file and returns it unlocked. -// The lock is acquired separately via Lock, so Open can hydrate without -// holding it. -func openLock(path string) (*lockedFile, error) { - f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0o644) - if err != nil { - return nil, fmt.Errorf("open lock file %s: %w", path, err) - } - return &lockedFile{f: f}, nil -} - -// Lock blocks until the OS-level exclusive flock is acquired. flock is -// reentrant on Linux per-fd, so calling Lock twice on the same lockedFile -// from the same goroutine is fine; calling it from two goroutines still -// serialises. -func (l *lockedFile) Lock() error { - if l == nil || l.f == nil { - return fmt.Errorf("cronstore: lock not initialised") - } - if err := syscall.Flock(int(l.f.Fd()), syscall.LOCK_EX); err != nil { - return fmt.Errorf("flock LOCK_EX: %w", err) - } - return nil -} - -// Unlock releases the exclusive flock. Idempotent in the sense that a -// second call after Close simply returns nil because the fd is closed. -func (l *lockedFile) Unlock() error { - if l == nil || l.f == nil { - return nil - } - if err := syscall.Flock(int(l.f.Fd()), syscall.LOCK_UN); err != nil { - return fmt.Errorf("flock LOCK_UN: %w", err) - } - return nil -} - -// Close closes the sidecar fd. The flock is released automatically by -// the kernel. -func (l *lockedFile) Close() error { - if l == nil || l.f == nil { - return nil - } - err := l.f.Close() - l.f = nil - return err -} diff --git a/internal/infrastructure/cronstore/lock_windows.go b/internal/infrastructure/cronstore/lock_windows.go deleted file mode 100644 index 823db691e..000000000 --- a/internal/infrastructure/cronstore/lock_windows.go +++ /dev/null @@ -1,53 +0,0 @@ -//go:build windows - -// flock is a POSIX primitive; Windows does not implement syscall.Flock -// the same way. Windows deployments of archied are not the supported -// surface (the deployment model in CLAUDE.md and deployments/*.toml is -// Linux + macOS), so this file deliberately fails closed: every Lock -// returns an error rather than silently degrading to "no lock". -// -// If a Windows deployment ever becomes supported, swap this for a real -// LockFileEx / UnlockFileEx implementation; the rest of the store does -// not need to change. - -package cronstore - -import ( - "errors" - "fmt" - "os" -) - -// lockedFile is a stub on Windows. The store still opens the sidecar -// file so the on-disk layout matches across platforms, but every Lock -// returns an explicit error. -type lockedFile struct { - f *os.File -} - -func openLock(path string) (*lockedFile, error) { - f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0o644) - if err != nil { - return nil, fmt.Errorf("open lock file %s: %w", path, err) - } - return &lockedFile{f: f}, nil -} - -// errLockUnsupported is wrapped in every Lock call so callers can match -// on it without parsing strings. -var errLockUnsupported = errors.New("cronstore: cross-process locking is unsupported on Windows") - -func (l *lockedFile) Lock() error { return errLockUnsupported } -func (l *lockedFile) Unlock() error { return nil } - -func (l *lockedFile) Close() error { - if l == nil || l.f == nil { - return nil - } - err := l.f.Close() - l.f = nil - return err -} - -// Keep imports alive when this file is built in isolation. -var _ = fmt.Sprintf diff --git a/internal/infrastructure/cronstore/process_lock.go b/internal/infrastructure/cronstore/process_lock.go deleted file mode 100644 index 49558d281..000000000 --- a/internal/infrastructure/cronstore/process_lock.go +++ /dev/null @@ -1,33 +0,0 @@ -package cronstore - -import "sync" - -// processLocks serialises Stores within one process. It exists because -// Linux flock is per file description, not per inode: two Store -// instances opening the same jobs.json.lock each get their own fd and -// their own independent flock state, so they would not block each -// other inside a single process. The cross-process lock handles the -// CLI-vs-daemon case; this map handles the in-process case. -// -// Keyed by absolute lock-file path so two unrelated cronstores in the -// same process do not contend. Entries are created lazily and live for -// the life of the process; the map is append-only and small in -// practice (one entry per distinct config dir). -var ( - processLocksMu sync.Mutex - processLocks = map[string]*sync.Mutex{} -) - -// processLockFor returns the per-lock-path mutex, creating it on first -// use. Callers must hold it for the duration of an operation that has -// already acquired the cross-process flock. -func processLockFor(path string) *sync.Mutex { - processLocksMu.Lock() - defer processLocksMu.Unlock() - if m, ok := processLocks[path]; ok { - return m - } - m := &sync.Mutex{} - processLocks[path] = m - return m -} diff --git a/internal/infrastructure/cronstore/schedule_duration_test.go b/internal/infrastructure/cronstore/schedule_duration_test.go deleted file mode 100644 index c1457c685..000000000 --- a/internal/infrastructure/cronstore/schedule_duration_test.go +++ /dev/null @@ -1,91 +0,0 @@ -package cronstore - -import ( - "encoding/json" - "strings" - "testing" - "time" -) - -// TestScheduleIntervalWritesADurationString pins the document form: the interval -// an operator edits must read as a duration, not as the nanosecond count that -// asked them to type 1800000000000. -func TestScheduleIntervalWritesADurationString(t *testing.T) { - encoded, err := json.Marshal(Schedule{Kind: ScheduleInterval, Interval: Duration(30 * time.Minute)}) - if err != nil { - t.Fatalf("marshal: %v", err) - } - if got := string(encoded); !strings.Contains(got, `"interval":"30m0s"`) { - t.Errorf("marshalled schedule = %s, want interval as the string \"30m0s\"", got) - } - if strings.Contains(string(encoded), "1800000000000") { - t.Errorf("marshalled schedule = %s, want no nanosecond count", encoded) - } -} - -// TestScheduleIntervalReadsEveryStoredForm covers what the store already holds. -// Every schedule written before this shape existed carries a nanosecond count, in -// both the control plane's schedules resource and this package's own jobs table, -// and both have to keep working without being re-saved first. -func TestScheduleIntervalReadsEveryStoredForm(t *testing.T) { - for _, tt := range []struct { - name string - document string - expected time.Duration - }{ - {"legacy nanosecond count", `{"kind":"interval","interval":1800000000000}`, 30 * time.Minute}, - {"duration string", `{"kind":"interval","interval":"30m"}`, 30 * time.Minute}, - {"duration string, canonical", `{"kind":"interval","interval":"30m0s"}`, 30 * time.Minute}, - } { - t.Run(tt.name, func(t *testing.T) { - var schedule Schedule - if err := json.Unmarshal([]byte(tt.document), &schedule); err != nil { - t.Fatalf("unmarshal %s: %v", tt.document, err) - } - if schedule.Interval.Std() != tt.expected { - t.Errorf("Interval = %v, want %v", schedule.Interval, tt.expected) - } - if err := schedule.Validate(); err != nil { - t.Errorf("Validate() = %v, want the decoded schedule to be usable", err) - } - }) - } -} - -// TestScheduleIntervalRefusesWhatItCannotUse keeps the tolerance above from -// becoming indifference: a duration that does not parse must fail loudly rather -// than decode as zero, because a zero interval is a job that never fires. -func TestScheduleIntervalRefusesWhatItCannotUse(t *testing.T) { - for _, tt := range []struct{ name, document string }{ - {"an unparseable duration", `{"kind":"interval","interval":"soon"}`}, - {"a fractional count", `{"kind":"interval","interval":1.5}`}, - {"a null interval", `{"kind":"interval","interval":null}`}, - } { - t.Run(tt.name, func(t *testing.T) { - var schedule Schedule - if err := json.Unmarshal([]byte(tt.document), &schedule); err == nil { - t.Fatalf("unmarshal(%s) = nil, want a refusal: a silent zero interval never fires", tt.document) - } - }) - } -} - -// TestScheduleIntervalRoundTripsThroughTheStore pins the shape the jobs table -// keeps, since it persists the spec as JSON rather than as columns. -func TestScheduleIntervalRoundTripsThroughTheStore(t *testing.T) { - spec := JobSpec{ID: "nightly", Schedule: Schedule{Kind: ScheduleInterval, Interval: Duration(90 * time.Minute)}} - encoded, err := json.Marshal(spec) - if err != nil { - t.Fatalf("marshal: %v", err) - } - if !strings.Contains(string(encoded), `"interval":"1h30m0s"`) { - t.Fatalf("marshalled spec = %s, want the interval as \"1h30m0s\"", encoded) - } - var decoded JobSpec - if err := json.Unmarshal(encoded, &decoded); err != nil { - t.Fatalf("unmarshal: %v", err) - } - if decoded.Schedule.Interval.Std() != 90*time.Minute { - t.Errorf("round-tripped Interval = %v, want 1h30m0s", decoded.Schedule.Interval) - } -} diff --git a/internal/infrastructure/cronstore/store.go b/internal/infrastructure/cronstore/store.go deleted file mode 100644 index 89df661ed..000000000 --- a/internal/infrastructure/cronstore/store.go +++ /dev/null @@ -1,665 +0,0 @@ -package cronstore - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "io/fs" - "os" - "path/filepath" - "strings" - "sync" - "sync/atomic" - "time" - - "github.com/samcharles93/archie-core/internal/domain/scheduling" -) - -// schemaVersion is stamped on every persisted file. Bump it on any -// breaking change to JobSpec, Schedule, Payload or envelope shape, AND -// on any additive change: DisallowUnknownFields means a future build -// that adds a new field to JobSpec must also bump schemaVersion so -// older builds keep rejecting the file rather than silently dropping -// the new field. -const schemaVersion = 2 - -// filePerm is the mode applied to the jobs.json file on write. Payloads -// and chat ids can carry operator-identifying information, so 0o600 is -// the default; operators who want it group-readable can chmod after. -const filePerm = 0o600 - -// JobSpec is one persisted cron job as the engine's view sees it, plus -// the bookkeeping fields the store computes (NextRun, LastRun, Created, -// Updated). The wire shape is the public API; the scheduling.Job returned -// to the engine is a stripped subset. -type JobSpec struct { - // ID is the unique identifier. Required, non-empty, and the merge - // key for Update and MarkRun. - ID string `json:"id"` - - // Pool is "parallel" or "sequential". Empty means sequential — the - // safer default, matching scheduling.Pool.resolve(). - Pool string `json:"pool,omitempty"` - - // Detail is a human-readable label rendered on events. - Detail string `json:"detail,omitempty"` - - // Kind selects which delivery runner handles this job. Empty means - // KindChat, so a job written before this field existed (schema - // version 1) keeps delivering as a chat message. The store does not - // interpret the value: an unrecognised kind is persisted verbatim and - // refused by the delivery router, which is the one place the mapping - // from kind to runner lives. - Kind string `json:"kind,omitempty"` - - // Schedule decides when this job is due. See schedule.go for the - // supported kinds and their arithmetic. - Schedule Schedule `json:"schedule"` - - // Target and Payload are placeholders for Slice 2 (delivery runner). - // They live on the wire now so jobs created ahead of Slice 2 do not - // need a schema migration when delivery lands. - Target Target `json:"target"` - Payload Payload `json:"payload"` - - // NextRun is the absolute moment the job will next fire. Computed - // at write time so the engine's hot path is a single comparison. - NextRun time.Time `json:"next_run"` - - // LastRun is the last run time reported by MarkRun, nil if the job - // has never run. Mutated only by MarkRun. The store does not know - // whether the run succeeded — that judgement belongs to the runner. - LastRun *time.Time `json:"last_run,omitempty"` - - // Created and Updated are stamped by Create / Update respectively. - // They are audit metadata; the engine does not read them. - Created time.Time `json:"created"` - Updated time.Time `json:"updated"` -} - -// Target identifies where the job's output goes (Slice 2 fills this in). -// Empty Target is valid — Slice 1 stores the field but no consumer reads -// it yet. -type Target struct { - // ChatID is the chat channel id (Telegram chat id, email address, - // webhook URL — depends on the channel). Empty defers to a default - // chosen by the delivery runner. - ChatID string `json:"chat_id,omitempty"` -} - -// Payload is the content sent at run time. Slice 2 fills this in; Slice 1 -// stores the text verbatim so the engine does not need to know about it. -type Payload struct { - // Text is the literal message body. Templating (cron blueprints) is - // a Slice 3 concern and is deliberately absent here. - Text string `json:"text,omitempty"` -} - -// Patch is the partial JobSpec Update merges over the existing record. A -// nil pointer field leaves the corresponding persisted field untouched; -// the non-nil pointer or non-zero value overwrites it. -// -// Schedule is replaced wholesale when the Patch's Schedule pointer is -// non-nil — schedules are not deep-merged, because each field of -// Schedule carries semantics (Kind, Interval, Cron, At) that have no -// meaningful "preserve" answer on their own. -type Patch struct { - Pool *string `json:"pool,omitempty"` - Detail *string `json:"detail,omitempty"` - Schedule *Schedule `json:"schedule,omitempty"` - Target *Target `json:"target,omitempty"` - Payload *Payload `json:"payload,omitempty"` -} - -// envelope is the on-disk shape. Kept private so the only public surface -// is JobSpec / Patch. -type envelope struct { - SchemaVersion int `json:"schema_version"` - Jobs []JobSpec `json:"jobs"` -} - -// Store is one open handle to the job file. It is safe for concurrent -// callers in the same process, and safe against concurrent CLI / daemon -// processes through the cross-process lock. -// -// All exported methods take a context for cancellation; reads and writes -// are short, so the ctx is plumbed through but rarely observed. -// -// After Close, every other method returns ErrClosed. Close itself is -// safe to call multiple times. -type Store struct { - path string // jobs.json - lock *lockedFile - - // closed is set by Close so concurrent callers fail fast rather than - // dereferencing a closed lock fd. - closed atomic.Bool - - // mu guards the in-memory copy that mirrors the file. Acquire it - // before reading or writing jobs; the cross-process lock guards - // the file itself. They must always be taken in the same order: - // cross-process lock first, then mu. - mu sync.Mutex - buf []JobSpec // in-memory mirror of the on-disk jobs array -} - -// ErrClosed is returned by every method after Close. It lets callers -// distinguish "I gave up" from a real error. -var ErrClosed = errors.New("cronstore: store is closed") - -// Open constructs a Store for the given jobs.json path. The file is -// created lazily on first write, but the parent directory is created -// immediately so a subsequent Create does not race against mkdir. -// -// Open validates the persisted file's schema version: a future version -// the build does not know returns ErrUnsupportedSchema rather than a -// silent best-effort decode. A corrupt or truncated file returns -// ErrCorruptFile. -func Open(path string) (*Store, error) { - if path == "" { - return nil, fmt.Errorf("%w: path must be non-empty", ErrInvalidSpec) - } - if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { - return nil, fmt.Errorf("cronstore: create parent directory: %w", err) - } - lock, err := openLock(path + ".lock") - if err != nil { - return nil, fmt.Errorf("cronstore: open lock: %w", err) - } - - s := &Store{ - path: path, - lock: lock, - } - - // Best-effort hydrate. An absent file is fine: the store will - // materialise it on first Create. - if err := s.hydrate(); err != nil { - _ = lock.Close() - return nil, err - } - return s, nil -} - -// Path returns the file the store reads and writes. Exposed for -// diagnostics and for the CLI to print the location it is editing. -func (s *Store) Path() string { return s.path } - -// Close releases the cross-process lock. After Close, every other -// method returns ErrClosed. Close is safe to call more than once. -func (s *Store) Close() error { - if !s.closed.CompareAndSwap(false, true) { - return nil - } - if s.lock == nil { - return nil - } - err := s.lock.Close() - s.lock = nil - return err -} - -// checkOpen returns ErrClosed when the store has been closed. Every -// public method calls this first so a Close mid-operation surfaces as a -// real error rather than a panic on a nil lock fd. -func (s *Store) checkOpen() error { - if s.closed.Load() { - return ErrClosed - } - return nil -} - -// Create adds a new job. Fails with ErrJobExists on a duplicate id, with -// ErrInvalidSpec on a structurally invalid spec. The store stamps -// Created / Updated to now and computes NextRun from the schedule. -func (s *Store) Create(ctx context.Context, spec JobSpec) error { - if err := ctx.Err(); err != nil { - return err - } - if err := s.checkOpen(); err != nil { - return err - } - if strings.TrimSpace(spec.ID) == "" { - return fmt.Errorf("%w: id must be non-empty", ErrInvalidSpec) - } - if err := spec.Schedule.Validate(); err != nil { - return err - } - if err := scheduling.Pool(spec.Pool).Validate(); err != nil { - return err - } - - if err := s.withLock(func() error { - for _, existing := range s.buf { - if existing.ID == spec.ID { - return fmt.Errorf("%w: id %q", ErrJobExists, spec.ID) - } - } - - now := time.Now().UTC() - // Honor a caller-supplied NextRun when present: the test suite - // needs to set up past/future due states deterministically, and - // the CLI may want to schedule a one-off for a specific time. - // Zero NextRun falls back to the schedule's first-run arithmetic. - if spec.NextRun.IsZero() { - next, err := spec.Schedule.firstRun(now) - if err != nil { - return err - } - spec.NextRun = next.UTC() - } - spec.Created = now - spec.Updated = now - spec.Schedule = spec.Schedule.resolve() - s.buf = append(s.buf, spec) - return s.writeLocked() - }); err != nil { - return err - } - return nil -} - -// Get returns the job by id. The second return value is false when the -// id is absent; the error is reserved for I/O failures. The returned -// JobSpec is a deep copy: callers can mutate LastRun and other fields -// without disturbing the store's in-memory mirror. -func (s *Store) Get(ctx context.Context, id string) (JobSpec, bool, error) { - if err := ctx.Err(); err != nil { - return JobSpec{}, false, err - } - if err := s.checkOpen(); err != nil { - return JobSpec{}, false, err - } - var ( - got JobSpec - ok bool - ) - err := s.withLock(func() error { - for _, j := range s.buf { - if j.ID == id { - got = deepCopyJob(j) - ok = true - return nil - } - } - return nil - }) - return got, ok, err -} - -// List returns a copy of every persisted job. Order is the on-disk -// order, which is also the creation order. Callers may mutate the -// returned slice and its elements without disturbing the store's -// in-memory mirror. -func (s *Store) List(ctx context.Context) ([]JobSpec, error) { - if err := ctx.Err(); err != nil { - return nil, err - } - if err := s.checkOpen(); err != nil { - return nil, err - } - var out []JobSpec - err := s.withLock(func() error { - out = make([]JobSpec, len(s.buf)) - for i := range s.buf { - out[i] = deepCopyJob(s.buf[i]) - } - return nil - }) - return out, err -} - -// Update applies a Patch over the existing record. Fields the Patch -// leaves nil are preserved verbatim; non-nil pointers and the Schedule -// are wholesale replacements. Schedule changes trigger a NextRun -// recomputation, since NextRun is derived from Schedule and LastRun. -// -// Returns ErrJobNotFound when the id is absent. -func (s *Store) Update(ctx context.Context, id string, patch Patch) error { - if err := ctx.Err(); err != nil { - return err - } - if err := s.checkOpen(); err != nil { - return err - } - if id == "" { - return fmt.Errorf("%w: id must be non-empty", ErrInvalidSpec) - } - if patch.Pool != nil { - if err := scheduling.Pool(*patch.Pool).Validate(); err != nil { - return err - } - } - if patch.Schedule != nil { - if err := patch.Schedule.Validate(); err != nil { - return err - } - } - return s.withLock(func() error { - for i := range s.buf { - if s.buf[i].ID != id { - continue - } - if err := applyPatch(&s.buf[i], patch); err != nil { - return err - } - s.buf[i].Updated = time.Now().UTC() - return s.writeLocked() - } - return fmt.Errorf("%w: id %q", ErrJobNotFound, id) - }) -} - -// applyPatch overwrites record's non-nil Patch fields in place, recomputing -// NextRun when Schedule changes. -func applyPatch(record *JobSpec, patch Patch) error { - if patch.Pool != nil { - record.Pool = *patch.Pool - } - if patch.Detail != nil { - record.Detail = *patch.Detail - } - if patch.Schedule != nil { - if err := applyScheduleChange(record, patch.Schedule); err != nil { - return err - } - } - if patch.Target != nil { - record.Target = *patch.Target - } - if patch.Payload != nil { - record.Payload = *patch.Payload - } - return nil -} - -// applyScheduleChange sets record's Schedule and recomputes NextRun, which -// is stale once Schedule changes. Recomputed from the last run if there is -// one, else as a first run from now — keeping it strictly future-of-now -// would re-fire a job the operator just shortened. -func applyScheduleChange(record *JobSpec, schedule *Schedule) error { - record.Schedule = schedule.resolve() - var next time.Time - var err error - if record.LastRun != nil { - next, err = record.Schedule.nextRun(*record.LastRun) - } else { - next, err = record.Schedule.firstRun(time.Now().UTC()) - } - if err != nil { - return err - } - record.NextRun = next.UTC() - return nil -} - -// Delete removes the job by id. Idempotent: deleting a missing id -// returns nil rather than ErrJobNotFound, so a CLI retry after a -// partial failure cannot stall the operator. -func (s *Store) Delete(ctx context.Context, id string) error { - if err := ctx.Err(); err != nil { - return err - } - if err := s.checkOpen(); err != nil { - return err - } - if id == "" { - return fmt.Errorf("%w: id must be non-empty", ErrInvalidSpec) - } - return s.withLock(func() error { - for i := range s.buf { - if s.buf[i].ID != id { - continue - } - s.buf = append(s.buf[:i], s.buf[i+1:]...) - return s.writeLocked() - } - return nil - }) -} - -// MarkRun records that id ran at runAt and advances NextRun to the -// schedule's next firing. It is the engine-side counterpart to Create: -// once the ticker has dispatched a job, the runner (or the engine -// itself, in tests) calls MarkRun to keep next_run fresh. -// -// Returns ErrJobNotFound when the id is absent and -// ErrScheduleUnsupported when the schedule kind has no defined -// "next after this run" semantics. -func (s *Store) MarkRun(ctx context.Context, id string, runAt time.Time) error { - if err := ctx.Err(); err != nil { - return err - } - if err := s.checkOpen(); err != nil { - return err - } - if id == "" { - return fmt.Errorf("%w: id must be non-empty", ErrInvalidSpec) - } - return s.withLock(func() error { - for i := range s.buf { - if s.buf[i].ID != id { - continue - } - next, err := s.buf[i].Schedule.nextRun(runAt) - if err != nil { - return err - } - t := runAt.UTC() - s.buf[i].LastRun = &t - s.buf[i].NextRun = next.UTC() - s.buf[i].Updated = time.Now().UTC() - return s.writeLocked() - } - return fmt.Errorf("%w: id %q", ErrJobNotFound, id) - }) -} - -// Due implements scheduling.JobSource. It returns the jobs whose -// NextRun is at or before now, in creation order. The returned -// scheduling.Job is the engine's projection of the persisted JobSpec: -// only the fields the engine relies on (ID, Pool, Detail) cross the -// boundary. Schedule / Payload / Target are deliberately not exposed: -// the engine does no arithmetic on them and has no business knowing -// they exist. -// -// A job with an unrecognised pool is skipped: Create / Update enforce -// the set, but a file edited by hand could carry an invalid value, and -// the engine cannot reason about a pool it has never heard of. The -// store does not crash on bad input — the same way it does not panic -// on a truncated file. -func (s *Store) Due(ctx context.Context, now time.Time) ([]scheduling.Job, error) { - if err := ctx.Err(); err != nil { - return nil, err - } - if err := s.checkOpen(); err != nil { - return nil, err - } - var due []scheduling.Job - err := s.withLock(func() error { - for _, j := range s.buf { - if j.NextRun.After(now) { - continue - } - pool := scheduling.Pool(j.Pool) - if err := pool.Validate(); err != nil { - continue - } - due = append(due, scheduling.Job{ - ID: j.ID, - Pool: pool, - Detail: j.Detail, - }) - } - return nil - }) - return due, err -} - -// withLock serialises access to the file. Callers receive a closure -// that runs with the cross-process lock held and the in-memory buffer -// freshly hydrated from disk; the closure's return value becomes -// withLock's return value. -// -// Three locks stack, always in this order: -// -// 1. processMu: keyed by lock-file path. Serialises every Store that -// points at the same jobs.json within one process. Necessary -// because Linux flock is per file description: two Store instances -// in the same process have independent fds and would otherwise -// proceed in parallel. fcntl / POSIX locks would not have this -// property — they would block — but flock is the documented -// primitive on Linux + macOS, so we compensate at this layer. -// 2. cross-process flock: serialises Store instances across -// processes (CLI vs daemon). flock alone is insufficient for -// in-process coordination (see #1). -// 3. s.mu: guards s.buf against same-Store concurrent goroutines. -// Redundant under #1 + #2 but cheap, and it documents the -// invariant for any reader trying to follow the locking story. -// -// After all three are held, the in-memory buffer is re-read from disk -// so a writer that just acquired the locks sees every change the -// previous writer committed. Without this, a Store that opened before -// any writes happened would carry an empty view forever and silently -// overwrite everything. -// -// Re-reading on every operation is acceptable for cron workloads: the -// file holds O(dozens) of jobs, not O(thousands), and operations are -// human-paced, not high-throughput. -func (s *Store) withLock(fn func() error) error { - processMu := processLockFor(s.path + ".lock") - processMu.Lock() - defer processMu.Unlock() - - if s.closed.Load() { - return ErrClosed - } - if err := s.lock.Lock(); err != nil { - return fmt.Errorf("cronstore: acquire lock: %w", err) - } - defer func() { _ = s.lock.Unlock() }() - - s.mu.Lock() - defer s.mu.Unlock() - - if err := s.hydrate(); err != nil { - return err - } - return fn() -} - -// hydrate reads the file into the in-memory buffer. Called from Open -// (no lock needed; nothing else can see the Store yet) and from -// withLock (with both locks held). -func (s *Store) hydrate() error { - data, err := os.ReadFile(s.path) - if err != nil { - if errors.Is(err, fs.ErrNotExist) { - return nil - } - return fmt.Errorf("%w: read %s: %w", ErrCorruptFile, s.path, err) - } - if len(bytes.TrimSpace(data)) == 0 { - return nil - } - var env envelope - dec := json.NewDecoder(bytes.NewReader(data)) - // Reject unknown fields so a typo on the operator's side does not - // silently land in a future field and shadow a real value. The - // cost is that any additive change to JobSpec MUST bump - // schemaVersion — see the comment on that constant. - dec.DisallowUnknownFields() - if err := dec.Decode(&env); err != nil { - return fmt.Errorf("%w: decode %s: %w", ErrCorruptFile, s.path, err) - } - if env.SchemaVersion == 0 { - return fmt.Errorf("%w: missing schema_version in %s", ErrCorruptFile, s.path) - } - if env.SchemaVersion > schemaVersion { - return fmt.Errorf("%w: file is version %d, this build understands up to %d", - ErrUnsupportedSchema, env.SchemaVersion, schemaVersion) - } - s.buf = env.Jobs - return nil -} - -// writeLocked persists the in-memory buffer to the file. Must be called -// with s.mu held and the cross-process lock held — both guarantees -// follow from withLock. Atomic write via temp-file + rename, so a -// crash mid-write leaves the previous file intact rather than a -// truncated mess the next Open would have to clean up. -func (s *Store) writeLocked() error { - env := envelope{ - SchemaVersion: schemaVersion, - Jobs: s.buf, - } - var buf bytes.Buffer - enc := json.NewEncoder(&buf) - // operator-readable file: do not escape <, >, &. The default - // encoding/json behaviour writes these as < etc, which is - // round-trip-safe but unreadable when a human opens jobs.json to - // understand why a job failed to fire. - enc.SetEscapeHTML(false) - if err := enc.Encode(env); err != nil { - return fmt.Errorf("cronstore: encode: %w", err) - } - if err := atomicWriteFile(s.path, buf.Bytes(), filePerm); err != nil { - return fmt.Errorf("cronstore: write %s: %w", s.path, err) - } - return nil -} - -// atomicWriteFile writes data to path via a sibling temp file and -// rename. Rename within a single filesystem is atomic on POSIX, so a -// concurrent reader either sees the previous file in full or the new -// one in full — never a half-written body. Sync before rename gives a -// durability guarantee: the inode's data is on disk before the -// directory entry switches over. -func atomicWriteFile(path string, data []byte, perm os.FileMode) error { - dir := filepath.Dir(path) - tmp, err := os.CreateTemp(dir, filepath.Base(path)+".tmp-*") - if err != nil { - return err - } - tmpName := tmp.Name() - defer func() { - // Best-effort cleanup on any failure path. The rename has - // either happened (and tmpName is gone) or it has not (and - // the temp file still needs to go). - _ = os.Remove(tmpName) - }() - if _, err := tmp.Write(data); err != nil { - _ = tmp.Close() - return err - } - if err := tmp.Sync(); err != nil { - _ = tmp.Close() - return err - } - if err := tmp.Close(); err != nil { - return err - } - if err := os.Chmod(tmpName, perm); err != nil { - return err - } - return os.Rename(tmpName, path) -} - -// deepCopyJob returns a JobSpec value independent of the source's -// pointer fields. JobSpec carries one pointer (LastRun); a shallow -// copy aliases it and lets callers mutate the store's in-memory state -// by writing through the returned value. -func deepCopyJob(j JobSpec) JobSpec { - out := j - if j.LastRun != nil { - t := *j.LastRun - out.LastRun = &t - } - if j.Schedule.At != nil { - t := *j.Schedule.At - out.Schedule.At = &t - } - return out -} diff --git a/internal/infrastructure/cronstore/store_review_test.go b/internal/infrastructure/cronstore/store_review_test.go deleted file mode 100644 index 19b87fea2..000000000 --- a/internal/infrastructure/cronstore/store_review_test.go +++ /dev/null @@ -1,485 +0,0 @@ -// Additional tests added in response to the Slice-1 adversarial review. -// They live in a separate file so the original spec file stays focused -// on the contract-level behaviour the engine depends on, and these -// cases stay grouped where the reviewer can re-read them as a unit. -package cronstore - -import ( - "bytes" - "context" - "errors" - "os" - "os/exec" - "path/filepath" - "sync" - "testing" - "time" - - "github.com/samcharles93/archie-core/internal/domain/scheduling" -) - -// --- C2 + P8: ErrCorruptFile is the right sentinel for bad files ------- - -func TestOpenTruncatedFileReturnsErrCorruptFile(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "jobs.json") - if err := os.WriteFile(path, []byte(`{"schema_version":1,"jobs":[{"id":"x","`), 0o600); err != nil { - t.Fatalf("WriteFile: %v", err) - } - _, err := Open(path) - if !errors.Is(err, ErrCorruptFile) { - t.Fatalf("Open on truncated file = %v, want ErrCorruptFile", err) - } -} - -func TestOpenGarbageFileReturnsErrCorruptFile(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "jobs.json") - if err := os.WriteFile(path, []byte("not json at all"), 0o600); err != nil { - t.Fatalf("WriteFile: %v", err) - } - _, err := Open(path) - if !errors.Is(err, ErrCorruptFile) { - t.Fatalf("Open on garbage file = %v, want ErrCorruptFile", err) - } -} - -func TestOpenEmptyFileIsTreatedAsNoJobs(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "jobs.json") - if err := os.WriteFile(path, []byte(" \n\t "), 0o600); err != nil { - t.Fatalf("WriteFile: %v", err) - } - s, err := Open(path) - if err != nil { - t.Fatalf("Open on whitespace-only file = %v, want nil", err) - } - defer s.Close() - jobs, _ := s.List(context.Background()) - if len(jobs) != 0 { - t.Errorf("List on whitespace-only file = %d jobs, want 0", len(jobs)) - } -} - -// --- C6 / P8: DisallowUnknownFields rejects hand-edited extras --------- - -func TestOpenRejectsUnknownFieldAtSameSchemaVersion(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "jobs.json") - // A future build might add a "tags" field to JobSpec. An older - // build with DisallowUnknownFields set must reject the file rather - // than silently dropping the field — that's the contract behind - // bumping schemaVersion on additive changes. - contents := `{"schema_version":1,"jobs":[{"id":"x","pool":"parallel","schedule":{"kind":"interval","interval":60000000000},"target":{},"payload":{},"next_run":"2026-08-28T12:00:00Z","created":"2026-08-28T12:00:00Z","updated":"2026-08-28T12:00:00Z","tags":["ops"]}]}` - if err := os.WriteFile(path, []byte(contents), 0o600); err != nil { - t.Fatalf("WriteFile: %v", err) - } - _, err := Open(path) - if !errors.Is(err, ErrCorruptFile) { - t.Fatalf("Open on file with unknown field = %v, want ErrCorruptFile", err) - } -} - -// --- P2: Get / List return deep copies ------------------------------- - -func TestGetReturnsDeepCopyOfLastRun(t *testing.T) { - s := newTestStore(t) - mustCreate(t, s, fixedIntervalSpec("x", time.Hour)) - first, _, _ := s.Get(context.Background(), "x") - if first.LastRun != nil { - t.Fatalf("LastRun on fresh create = %v, want nil", first.LastRun) - } - runAt := time.Now().UTC().Add(-time.Minute) - if err := s.MarkRun(context.Background(), "x", runAt); err != nil { - t.Fatalf("MarkRun: %v", err) - } - got, _, _ := s.Get(context.Background(), "x") - if got.LastRun == nil { - t.Fatal("LastRun is nil after MarkRun") - } - // Caller mutates the returned LastRun. The store's in-memory - // value must NOT change — a future Get must still report the - // original runAt. - *got.LastRun = time.Date(2099, 1, 1, 0, 0, 0, 0, time.UTC) - got2, _, _ := s.Get(context.Background(), "x") - if !got2.LastRun.Equal(runAt) { - t.Errorf("after caller mutation, Get returned LastRun=%v, want %v", got2.LastRun, runAt) - } -} - -func TestListReturnsDeepCopies(t *testing.T) { - s := newTestStore(t) - mustCreate(t, s, fixedIntervalSpec("x", time.Hour)) - mustCreate(t, s, fixedIntervalSpec("y", time.Hour)) - - list1, _ := s.List(context.Background()) - if len(list1) != 2 { - t.Fatalf("List returned %d jobs, want 2", len(list1)) - } - // Caller mutates the returned slice (deletes an entry) and a job's - // string fields. Neither must touch the store's in-memory mirror. - list1[0].Detail = "tampered" - - list2, _ := s.List(context.Background()) - if len(list2) != 2 { - t.Errorf("after caller truncated list, second List returned %d jobs, want 2", len(list2)) - } - for _, j := range list2 { - if j.Detail == "tampered" { - t.Errorf("caller's Detail mutation leaked into the store: %+v", j) - } - } -} - -// --- P3: pool validation in Create / Update --------------------------- - -func TestCreateRejectsInvalidPool(t *testing.T) { - s := newTestStore(t) - spec := fixedIntervalSpec("x", time.Minute) - spec.Pool = "paralle" // typo - if err := s.Create(context.Background(), spec); err == nil { - t.Fatal("Create with invalid pool succeeded; want error") - } -} - -func TestUpdateRejectsInvalidPool(t *testing.T) { - s := newTestStore(t) - mustCreate(t, s, fixedIntervalSpec("x", time.Minute)) - if err := s.Update(context.Background(), "x", Patch{Pool: new("paralle")}); err == nil { - t.Fatal("Update with invalid pool succeeded; want error") - } -} - -// --- P5 / C1 / P6: Once schedule behaviour + MarkRun sentinel --------- - -func TestCreateAcceptsOnceSchedule(t *testing.T) { - s := newTestStore(t) - at := time.Now().Add(time.Hour) - spec := JobSpec{ - ID: "one-off", - Schedule: Schedule{ - Kind: ScheduleOnce, - At: &at, - }, - } - if err := s.Create(context.Background(), spec); err != nil { - t.Fatalf("Create with Once schedule: %v", err) - } -} - -func TestDueFiresOnceAtConfiguredTime(t *testing.T) { - s := newTestStore(t) - at := time.Date(2026, 8, 28, 12, 0, 0, 0, time.UTC) - spec := JobSpec{ - ID: "one-off", - Schedule: Schedule{ - Kind: ScheduleOnce, - At: &at, - }, - } - if err := s.Create(context.Background(), spec); err != nil { - t.Fatalf("Create: %v", err) - } - - due, err := s.Due(context.Background(), at) - if err != nil { - t.Fatalf("Due: %v", err) - } - if len(due) != 1 || due[0].ID != "one-off" { - t.Errorf("Due(at) = %+v, want one-off", due) - } - due, _ = s.Due(context.Background(), at.Add(-time.Second)) - if len(due) != 0 { - t.Errorf("Due(at - 1s) = %+v, want empty", due) - } -} - -func TestMarkRunOnOnceReturnsErrScheduleUnsupported(t *testing.T) { - s := newTestStore(t) - at := time.Now().Add(time.Hour) - spec := JobSpec{ - ID: "one-off", - Schedule: Schedule{ - Kind: ScheduleOnce, - At: &at, - }, - } - if err := s.Create(context.Background(), spec); err != nil { - t.Fatalf("Create: %v", err) - } - err := s.MarkRun(context.Background(), "one-off", time.Now()) - if !errors.Is(err, ErrScheduleUnsupported) { - t.Fatalf("MarkRun on once = %v, want ErrScheduleUnsupported", err) - } -} - -// --- P8: Due drops unknown pool values silently ----------------------- - -func TestDueSilentlySkipsUnknownPool(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "jobs.json") - s, err := Open(path) - if err != nil { - t.Fatalf("Open: %v", err) - } - defer s.Close() - // Hand-edit the file: bypass Create's validation by writing a - // job with an invalid pool directly to disk. - past := time.Now().Add(-time.Minute).UTC().Format(time.RFC3339Nano) - contents := `{"schema_version":1,"jobs":[{"id":"badpool","pool":"paralle","schedule":{"kind":"interval","interval":60000000000},"target":{},"payload":{},"next_run":"` + past + `","created":"` + past + `","updated":"` + past + `"}]}` - if err := os.WriteFile(path, []byte(contents), 0o600); err != nil { - t.Fatalf("WriteFile: %v", err) - } - // Drop the in-memory mirror so the next read picks up the file. - if err := s.hydrate(); err != nil { - t.Fatalf("hydrate: %v", err) - } - due, err := s.Due(context.Background(), time.Now()) - if err != nil { - t.Fatalf("Due: %v", err) - } - if len(due) != 0 { - t.Errorf("Due returned %d jobs for an invalid pool, want 0", len(due)) - } - // But Get still sees the job — Create/Update validation is the - // gate that prevents this in normal operation; a hand-edited - // file remains visible through CRUD. - if _, ok, _ := s.Get(context.Background(), "badpool"); !ok { - t.Error("Get returned ok=false for a hand-edited job with bad pool") - } -} - -// --- P10: UTF-8 + quote + newline round-trip -------------------------- - -func TestCreateAndGetRoundTripWithUnicodeAndQuotes(t *testing.T) { - s := newTestStore(t) - spec := JobSpec{ - ID: "ops-standup", - Pool: string(scheduling.PoolSequential), - Detail: "Hello\n\"世界\"\\back", - Schedule: Schedule{ - Kind: ScheduleInterval, - Interval: Duration(30 * time.Minute), - }, - Payload: Payload{Text: " & 中文"}, - } - mustCreate(t, s, spec) - got, ok, err := s.Get(context.Background(), spec.ID) - if err != nil { - t.Fatalf("Get: %v", err) - } - if !ok { - t.Fatal("Get returned ok=false") - } - if got.Detail != spec.Detail { - t.Errorf("Detail = %q, want %q", got.Detail, spec.Detail) - } - if got.Payload.Text != spec.Payload.Text { - t.Errorf("Payload.Text = %q, want %q", got.Payload.Text, spec.Payload.Text) - } -} - -// --- C3: operator-readable file does not HTML-escape ------------------- - -func TestFileIsOperatorReadableNoHTMLEscape(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "jobs.json") - s, _ := Open(path) - defer s.Close() - spec := fixedIntervalSpec("x", time.Minute) - spec.Payload.Text = "" - mustCreate(t, s, spec) - _ = s.Close() - - data, err := os.ReadFile(path) - if err != nil { - t.Fatalf("ReadFile: %v", err) - } - // With SetEscapeHTML(false), < > & appear as the literal characters. - // The escaped form would be <, >, & — none of which - // is the printable payload. Assert the literal is there. - if !bytes.Contains(data, []byte(`""`)) { - t.Errorf("payload was HTML-escaped; want operator-readable literal: %s", data) - } -} - -// --- C4: file is 0o600, not 0o644 -------------------------------------- - -func TestFileIsCreatedWithOwnerOnlyPermissions(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "jobs.json") - s, _ := Open(path) - defer s.Close() - mustCreate(t, s, fixedIntervalSpec("x", time.Minute)) - _ = s.Close() - - info, err := os.Stat(path) - if err != nil { - t.Fatalf("Stat: %v", err) - } - perm := info.Mode().Perm() - if perm != 0o600 { - t.Errorf("jobs.json perm = %#o, want 0o600", perm) - } -} - -// --- C5: cross-process flock via os/exec ------------------------------ - -// TestCrossProcessFlockBlocksParentWhileChildHoldsLock runs a child -// process that grabs the cross-process flock and sleeps; the parent -// then tries to take the same lock and must block for at least the -// child's sleep duration. Without the flock, the parent's Create -// would return immediately and the test would fail. -func TestCrossProcessFlockBlocksParentWhileChildHoldsLock(t *testing.T) { - if testing.Short() { - t.Skip("subprocess test") - } - if _, err := exec.LookPath("go"); err != nil { - t.Skip("no go toolchain available for subprocess test") - } - dir := t.TempDir() - path := filepath.Join(dir, "jobs.json") - ctx := t.Context() - - // Build a tiny helper that opens the same file and holds the - // flock. The helper is built once per run; the cost is paid only - // when this test runs. - helperSrc := filepath.Join(dir, "helper.go") - if err := os.WriteFile(helperSrc, []byte(` -package main - -import ( - "fmt" - "os" - "syscall" - "time" -) - -func main() { - f, err := os.OpenFile(os.Args[1]+".lock", os.O_RDWR|os.O_CREATE, 0o644) - if err != nil { fmt.Println("open:", err); os.Exit(2) } - if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil { - fmt.Println("flock:", err); os.Exit(3) - } - fmt.Println("locked") - // Signal readiness, then hold the lock long enough for the - // parent to observe the block. - time.Sleep(2 * time.Second) - _ = f.Close() -} -`), 0o644); err != nil { - t.Fatalf("WriteFile helper: %v", err) - } - helperBin := filepath.Join(dir, "helper") - if out, err := exec.CommandContext(ctx, "go", "build", "-o", helperBin, helperSrc).CombinedOutput(); err != nil { - t.Fatalf("go build helper: %v\n%s", err, out) - } - - // Launch the child in the background. - child := exec.CommandContext(ctx, helperBin, path) - child.Stdout = os.Stdout - child.Stderr = os.Stderr - if err := child.Start(); err != nil { - t.Fatalf("child Start: %v", err) - } - defer func() { - _ = child.Process.Kill() - _ = child.Wait() - }() - - // Give the child a moment to acquire the lock. - time.Sleep(300 * time.Millisecond) - - // From the parent, open the same Store and call Create. Without - // flock, Create would return instantly; with flock, it must - // block until the child releases. - parent, err := Open(path) - if err != nil { - t.Fatalf("parent Open: %v", err) - } - defer parent.Close() - - start := time.Now() - if err := parent.Create(context.Background(), fixedIntervalSpec("from-parent", time.Minute)); err != nil { - t.Fatalf("parent Create: %v", err) - } - elapsed := time.Since(start) - - // The child held the lock for ~2s. The parent's Create must have - // blocked for at least the overlap between its start and the - // child's release — conservatively, >1s. - if elapsed < time.Second { - t.Errorf("parent Create returned in %v; expected to block on the child's flock", elapsed) - } -} - -// --- P1: Close mid-operation returns ErrClosed ------------------------ - -func TestMethodsReturnErrClosedAfterClose(t *testing.T) { - s := newTestStore(t) - mustCreate(t, s, fixedIntervalSpec("x", time.Minute)) - if err := s.Close(); err != nil { - t.Fatalf("Close: %v", err) - } - // Every public method must surface ErrClosed, not panic on the - // nil lock fd. - tests := []struct { - name string - fn func() error - }{ - {"Create", func() error { return s.Create(context.Background(), fixedIntervalSpec("y", time.Minute)) }}, - {"Get", func() error { _, _, err := s.Get(context.Background(), "x"); return err }}, - {"List", func() error { _, err := s.List(context.Background()); return err }}, - {"Update", func() error { return s.Update(context.Background(), "x", Patch{Detail: new("z")}) }}, - {"Delete", func() error { return s.Delete(context.Background(), "x") }}, - {"MarkRun", func() error { return s.MarkRun(context.Background(), "x", time.Now()) }}, - {"Due", func() error { _, err := s.Due(context.Background(), time.Now()); return err }}, - } - // Close itself is documented as idempotent (returns nil on the - // second call), so it does NOT appear in the ErrClosed list above. - if err := s.Close(); err != nil { - t.Errorf("second Close = %v, want nil", err) - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - err := tc.fn() - if !errors.Is(err, ErrClosed) { - t.Errorf("%s after Close = %v, want ErrClosed", tc.name, err) - } - }) - } -} - -// --- P8: same-path stores share the in-process mutex ------------------ - -func TestTwoStoresForSamePathSerialiseInProcess(t *testing.T) { - // This is the test that fails without process_lock.go. With it, - // 50/50 writes from each side land; without it, half the data - // disappears. Mirrors TestConcurrentWritersAreSerialised but in - // its own right so the regression is named. - dir := t.TempDir() - path := filepath.Join(dir, "jobs.json") - a, _ := Open(path) - defer a.Close() - b, _ := Open(path) - defer b.Close() - - var wg sync.WaitGroup - for i := range 25 { - wg.Add(2) - go func(i int) { - defer wg.Done() - _ = a.Create(context.Background(), fixedIntervalSpec("a"+itoa(i), time.Minute)) - }(i) - go func(i int) { - defer wg.Done() - _ = b.Create(context.Background(), fixedIntervalSpec("b"+itoa(i), time.Minute)) - }(i) - } - wg.Wait() - - jobs, _ := a.List(context.Background()) - if len(jobs) != 50 { - t.Errorf("after 50 concurrent writes from two stores, %d persisted, want 50", len(jobs)) - } -} diff --git a/internal/infrastructure/cronstore/store_test.go b/internal/infrastructure/cronstore/store_test.go deleted file mode 100644 index ec3694e34..000000000 --- a/internal/infrastructure/cronstore/store_test.go +++ /dev/null @@ -1,512 +0,0 @@ -// Package cronstore is the persistent job store backing the scheduling -// ticker engine. Tests in this file are the spec: an implementing agent -// must make every test in this file pass WITHOUT modifying the tests -// themselves. -// -// The store's only contract with the engine is scheduling.JobSource: one -// file-backed JSON document, cross-process locked, with version-stamped -// schema so future slices can extend it without breaking old files. -package cronstore - -import ( - "context" - "errors" - "os" - "path/filepath" - "strconv" - "strings" - "sync" - "testing" - "time" - - "github.com/samcharles93/archie-core/internal/domain/scheduling" -) - -// helpers - -func newTestStore(t *testing.T) *Store { - t.Helper() - dir := t.TempDir() - path := filepath.Join(dir, "jobs.json") - s, err := Open(path) - if err != nil { - t.Fatalf("Open(%s): %v", path, err) - } - t.Cleanup(func() { _ = s.Close() }) - return s -} - -func mustCreate(t *testing.T, s *Store, spec JobSpec) { - t.Helper() - if err := s.Create(context.Background(), spec); err != nil { - t.Fatalf("Create(%q): %v", spec.ID, err) - } -} - -func fixedIntervalSpec(id string, every time.Duration) JobSpec { - return JobSpec{ - ID: id, - Pool: string(scheduling.PoolParallel), - Detail: "test job " + id, - Schedule: Schedule{ - Kind: ScheduleInterval, - Interval: Duration(every), - }, - Payload: Payload{Text: "hello " + id}, - } -} - -// --- open / close ------------------------------------------------------------ - -func TestOpenMissingFileReturnsEmpty(t *testing.T) { - dir := t.TempDir() - s, err := Open(filepath.Join(dir, "jobs.json")) - if err != nil { - t.Fatalf("Open: %v", err) - } - defer s.Close() - - jobs, err := s.List(context.Background()) - if err != nil { - t.Fatalf("List: %v", err) - } - if len(jobs) != 0 { - t.Errorf("List on missing file = %d jobs, want 0", len(jobs)) - } -} - -func TestOpenCreatesParentDirectory(t *testing.T) { - dir := t.TempDir() - deep := filepath.Join(dir, "a", "b", "c", "jobs.json") - s, err := Open(deep) - if err != nil { - t.Fatalf("Open with missing parent: %v", err) - } - defer s.Close() - mustCreate(t, s, fixedIntervalSpec("x", time.Minute)) - if _, err := os.Stat(deep); err != nil { - t.Fatalf("expected file to exist at %s: %v", deep, err) - } -} - -// --- schema versioning ------------------------------------------------------- - -func TestPersistedFileHasVersionStamp(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "jobs.json") - s, err := Open(path) - if err != nil { - t.Fatalf("Open: %v", err) - } - mustCreate(t, s, fixedIntervalSpec("x", time.Minute)) - _ = s.Close() - - data, err := os.ReadFile(path) - if err != nil { - t.Fatalf("ReadFile: %v", err) - } - // Asserted against the constant, not a literal: the stamp must track - // schemaVersion through every bump (Kind was the first), and a - // hardcoded value here would fail on the bump as a false alarm - // instead of guarding the property it names. - want := `"schema_version":` + strconv.Itoa(schemaVersion) - if !strings.Contains(string(data), want) { - t.Errorf("file does not contain %s stamp: %s", want, data) - } -} - -func TestOpenRejectsUnknownSchemaVersion(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "jobs.json") - if err := os.WriteFile(path, []byte(`{"schema_version":999,"jobs":[]}`), 0o644); err != nil { - t.Fatalf("WriteFile: %v", err) - } - if _, err := Open(path); err == nil { - t.Fatal("Open with future schema version succeeded; want error") - } else if !errors.Is(err, ErrUnsupportedSchema) { - t.Errorf("error = %v, want ErrUnsupportedSchema", err) - } -} - -// --- create / read ----------------------------------------------------------- - -func TestCreateAndGetRoundTrip(t *testing.T) { - s := newTestStore(t) - want := JobSpec{ - ID: "morning-status", - Pool: string(scheduling.PoolSequential), - Detail: "morning status", - Schedule: Schedule{ - Kind: ScheduleInterval, - Interval: Duration(30 * time.Minute), - }, - Payload: Payload{Text: "status please"}, - } - mustCreate(t, s, want) - - got, ok, err := s.Get(context.Background(), want.ID) - if err != nil { - t.Fatalf("Get: %v", err) - } - if !ok { - t.Fatal("Get returned ok=false") - } - if got.ID != want.ID { - t.Errorf("ID = %q, want %q", got.ID, want.ID) - } - if got.Pool != want.Pool { - t.Errorf("Pool = %q, want %q", got.Pool, want.Pool) - } - if got.Detail != want.Detail { - t.Errorf("Detail = %q, want %q", got.Detail, want.Detail) - } - if got.Schedule.Kind != want.Schedule.Kind { - t.Errorf("Schedule.Kind = %q, want %q", got.Schedule.Kind, want.Schedule.Kind) - } - if got.Schedule.Interval != want.Schedule.Interval { - t.Errorf("Schedule.Interval = %v, want %v", got.Schedule.Interval, want.Schedule.Interval) - } - if got.Payload.Text != want.Payload.Text { - t.Errorf("Payload.Text = %q, want %q", got.Payload.Text, want.Payload.Text) - } - if got.NextRun.IsZero() { - t.Error("NextRun is zero; Create must compute it for an interval job") - } - if got.Created.IsZero() { - t.Error("Created is zero; Create must stamp it") - } - if got.Updated.IsZero() { - t.Error("Updated is zero; Create must stamp it") - } - if !got.Created.Equal(got.Updated) { - t.Errorf("Created %v != Updated %v on a fresh create", got.Created, got.Updated) - } -} - -func TestGetMissingReturnsNotFound(t *testing.T) { - s := newTestStore(t) - _, ok, err := s.Get(context.Background(), "nope") - if err != nil { - t.Fatalf("Get: %v", err) - } - if ok { - t.Error("Get of missing id returned ok=true") - } -} - -func TestCreateRejectsEmptyID(t *testing.T) { - s := newTestStore(t) - if err := s.Create(context.Background(), fixedIntervalSpec("", time.Minute)); err == nil { - t.Fatal("Create with empty id succeeded; want error") - } -} - -func TestCreateRejectsUnknownScheduleKind(t *testing.T) { - s := newTestStore(t) - spec := fixedIntervalSpec("x", time.Minute) - spec.Schedule.Kind = "magic" - if err := s.Create(context.Background(), spec); err == nil { - t.Fatal("Create with unknown schedule kind succeeded; want error") - } -} - -func TestCreateRejectsDuplicateID(t *testing.T) { - s := newTestStore(t) - mustCreate(t, s, fixedIntervalSpec("dup", time.Minute)) - if err := s.Create(context.Background(), fixedIntervalSpec("dup", time.Hour)); err == nil { - t.Fatal("Create with duplicate id succeeded; want error") - } -} - -// --- update with merge semantics -------------------------------------------- - -func TestUpdatePreservesFieldsNotInPatch(t *testing.T) { - s := newTestStore(t) - mustCreate(t, s, JobSpec{ - ID: "patch-me", - Pool: string(scheduling.PoolParallel), - Detail: "original detail", - Schedule: Schedule{ - Kind: ScheduleInterval, - Interval: Duration(10 * time.Minute), - }, - Payload: Payload{Text: "original payload"}, - }) - - if err := s.Update(context.Background(), "patch-me", Patch{ - Detail: new("updated detail"), - }); err != nil { - t.Fatalf("Update: %v", err) - } - - got, _, _ := s.Get(context.Background(), "patch-me") - if got.Detail != "updated detail" { - t.Errorf("Detail = %q, want %q", got.Detail, "updated detail") - } - // Untouched fields must survive. - if got.Pool != string(scheduling.PoolParallel) { - t.Errorf("Pool = %q, want %q (merge must preserve)", got.Pool, scheduling.PoolParallel) - } - if got.Schedule.Interval.Std() != 10*time.Minute { - t.Errorf("Schedule.Interval = %v, want %v (merge must preserve)", got.Schedule.Interval, 10*time.Minute) - } - if got.Payload.Text != "original payload" { - t.Errorf("Payload.Text = %q, want %q (merge must preserve)", got.Payload.Text, "original payload") - } -} - -func TestUpdateAdvancesUpdatedTimestamp(t *testing.T) { - s := newTestStore(t) - mustCreate(t, s, fixedIntervalSpec("ts", time.Minute)) - first, _, _ := s.Get(context.Background(), "ts") - - // Sleep long enough that Updated must move forward even on coarse - // clocks. mtime resolution on some filesystems is the second. - time.Sleep(1100 * time.Millisecond) - - if err := s.Update(context.Background(), "ts", Patch{Detail: new("new")}); err != nil { - t.Fatalf("Update: %v", err) - } - got, _, _ := s.Get(context.Background(), "ts") - if !got.Updated.After(first.Updated) { - t.Errorf("Updated did not advance: first=%v got=%v", first.Updated, got.Updated) - } -} - -func TestUpdateRecomputesNextRunWhenScheduleChanges(t *testing.T) { - s := newTestStore(t) - mustCreate(t, s, fixedIntervalSpec("rs", time.Hour)) - first, _, _ := s.Get(context.Background(), "rs") - - // Change the interval; NextRun must move. - if err := s.Update(context.Background(), "rs", Patch{ - Schedule: &Schedule{Kind: ScheduleInterval, Interval: Duration(2 * time.Minute)}, - }); err != nil { - t.Fatalf("Update: %v", err) - } - got, _, _ := s.Get(context.Background(), "rs") - if got.Schedule.Interval.Std() != 2*time.Minute { - t.Errorf("Schedule.Interval = %v, want %v", got.Schedule.Interval, 2*time.Minute) - } - if !got.NextRun.Before(first.NextRun) { - t.Errorf("NextRun did not move earlier after shortening interval: first=%v got=%v", first.NextRun, got.NextRun) - } -} - -func TestUpdateMissingReturnsNotFound(t *testing.T) { - s := newTestStore(t) - if err := s.Update(context.Background(), "ghost", Patch{Detail: new("x")}); err == nil { - t.Fatal("Update of missing id succeeded; want error") - } -} - -// --- delete ------------------------------------------------------------------ - -func TestDeleteRemovesJob(t *testing.T) { - s := newTestStore(t) - mustCreate(t, s, fixedIntervalSpec("bye", time.Minute)) - if err := s.Delete(context.Background(), "bye"); err != nil { - t.Fatalf("Delete: %v", err) - } - if _, ok, _ := s.Get(context.Background(), "bye"); ok { - t.Error("Get after Delete returned ok=true") - } -} - -func TestDeleteMissingIsIdempotent(t *testing.T) { - s := newTestStore(t) - // First delete is a no-op on a missing id; second delete must not panic - // and must not return an error. CLI retries should be safe. - if err := s.Delete(context.Background(), "never-existed"); err != nil { - t.Errorf("Delete(missing) = %v, want nil", err) - } - if err := s.Delete(context.Background(), "never-existed"); err != nil { - t.Errorf("Delete(missing) again = %v, want nil", err) - } -} - -// --- list -------------------------------------------------------------------- - -func TestListReturnsAllJobs(t *testing.T) { - s := newTestStore(t) - mustCreate(t, s, fixedIntervalSpec("a", time.Minute)) - mustCreate(t, s, fixedIntervalSpec("b", time.Minute)) - mustCreate(t, s, fixedIntervalSpec("c", time.Minute)) - - jobs, err := s.List(context.Background()) - if err != nil { - t.Fatalf("List: %v", err) - } - if len(jobs) != 3 { - t.Errorf("List returned %d jobs, want 3", len(jobs)) - } -} - -// --- JobSource.Due ----------------------------------------------------------- - -func TestDueReturnsOnlyJobsWhoseNextRunIsInThePast(t *testing.T) { - s := newTestStore(t) - now := time.Date(2026, 8, 28, 12, 0, 0, 0, time.UTC) - - // past: due - past := fixedIntervalSpec("past", 30*time.Minute) - past.NextRun = now.Add(-time.Minute) - // future: not due - future := fixedIntervalSpec("future", 30*time.Minute) - future.NextRun = now.Add(time.Hour) - mustCreate(t, s, past) - mustCreate(t, s, future) - - got, err := s.Due(context.Background(), now) - if err != nil { - t.Fatalf("Due: %v", err) - } - if len(got) != 1 { - t.Fatalf("Due returned %d jobs, want 1: %+v", len(got), got) - } - if got[0].ID != "past" { - t.Errorf("due job = %q, want %q", got[0].ID, "past") - } - if got[0].Pool != scheduling.PoolParallel { - t.Errorf("due job pool = %q, want %q", got[0].Pool, scheduling.PoolParallel) - } - if got[0].Detail != "test job past" { - t.Errorf("due job detail = %q, want %q", got[0].Detail, "test job past") - } -} - -func TestDueOnEmptyStoreReturnsEmptySlice(t *testing.T) { - s := newTestStore(t) - got, err := s.Due(context.Background(), time.Now()) - if err != nil { - t.Fatalf("Due: %v", err) - } - if len(got) != 0 { - t.Errorf("Due on empty = %d jobs, want 0", len(got)) - } -} - -func TestImplementsSchedulingJobSource(t *testing.T) { - // Static, compile-time check: *Store must satisfy scheduling.JobSource - // so the ticker engine can poll it without knowing the file layout. - var _ scheduling.JobSource = (*Store)(nil) -} - -func TestMarkRunAdvancesNextRun(t *testing.T) { - s := newTestStore(t) - mustCreate(t, s, fixedIntervalSpec("recurring", time.Hour)) - - first, _, _ := s.Get(context.Background(), "recurring") - runAt := first.NextRun.Add(time.Minute) // pretend engine ran a bit late - if err := s.MarkRun(context.Background(), "recurring", runAt); err != nil { - t.Fatalf("MarkRun: %v", err) - } - got, _, _ := s.Get(context.Background(), "recurring") - if got.LastRun == nil { - t.Fatal("LastRun is nil after MarkRun") - } - if !got.LastRun.Equal(runAt) { - t.Errorf("LastRun = %v, want %v", got.LastRun, runAt) - } - wantNext := runAt.Add(time.Hour) - if !got.NextRun.Equal(wantNext) { - t.Errorf("NextRun = %v, want %v (last_run + interval)", got.NextRun, wantNext) - } -} - -func TestMarkRunMissingReturnsNotFound(t *testing.T) { - s := newTestStore(t) - if err := s.MarkRun(context.Background(), "ghost", time.Now()); err == nil { - t.Fatal("MarkRun of missing id succeeded; want error") - } -} - -// --- partial-write handling -------------------------------------------------- - -func TestOpenTruncatedFileReturnsExplicitError(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "jobs.json") - // Header present, body cut mid-array. Must not panic on read; the - // engine surfaces this as phase="source" KindJobError, not a crash. - if err := os.WriteFile(path, []byte(`{"schema_version":1,"jobs":[{"id":"x","`), 0o644); err != nil { - t.Fatalf("WriteFile: %v", err) - } - _, err := Open(path) - if err == nil { - t.Fatal("Open on truncated file succeeded; want explicit error") - } - if errors.Is(err, ErrUnsupportedSchema) { - t.Errorf("error = ErrUnsupportedSchema; want a corrupt-file error") - } -} - -func TestOpenGarbageFileReturnsExplicitError(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "jobs.json") - if err := os.WriteFile(path, []byte("not json at all"), 0o644); err != nil { - t.Fatalf("WriteFile: %v", err) - } - if _, err := Open(path); err == nil { - t.Fatal("Open on garbage file succeeded; want explicit error") - } -} - -// --- cross-process locking --------------------------------------------------- - -func TestConcurrentWritersAreSerialised(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "jobs.json") - a, err := Open(path) - if err != nil { - t.Fatalf("Open a: %v", err) - } - defer a.Close() - b, err := Open(path) - if err != nil { - t.Fatalf("Open b: %v", err) - } - defer b.Close() - - // 50 writes from each writer, on the same file, interleaved. Without - // the lock, two writers would race and the file would end up either - // truncated or holding a non-deterministic subset. The lock makes - // each writer wait, so all 100 distinct ids end up persisted. - var wg sync.WaitGroup - for i := range 50 { - wg.Add(2) - go func(i int) { - defer wg.Done() - id := "a" + itoa(i) - _ = a.Create(context.Background(), fixedIntervalSpec(id, time.Minute)) - }(i) - go func(i int) { - defer wg.Done() - id := "b" + itoa(i) - _ = b.Create(context.Background(), fixedIntervalSpec(id, time.Minute)) - }(i) - } - wg.Wait() - - jobs, err := a.List(context.Background()) - if err != nil { - t.Fatalf("List: %v", err) - } - if len(jobs) != 100 { - t.Errorf("after 100 concurrent writes, %d jobs persisted, want 100", len(jobs)) - } -} - -// itoa avoids strconv import in tests. -func itoa(n int) string { - if n == 0 { - return "0" - } - var buf [20]byte - i := len(buf) - for n > 0 { - i-- - buf[i] = byte('0' + n%10) - n /= 10 - } - return string(buf[i:]) -} From ba52a34d83d9103db35d0b682308a6f02d642ef5 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Wed, 23 Sep 2026 02:19:28 +1000 Subject: [PATCH 08/20] feat(edastore)!: move event capture and bindings onto PocketBase collections The event-capture tables (captures, mappings, bindings, binding_dispatches, playbook_dispatches) now live in PocketBase-owned collections and their ids are opaque record ids rather than autoincrement integers. The task lifecycle stays on SQLite: tasks/transitions/events carry FIFO claim ordering, id-cursor pagination and the task-grant map, none of which gain anything from being hand-editable. binding_dispatches.task_id is the single seam and stays an integer addressing that store. BREAKING: CapturedEvent.Id, Mapping.Id, Binding.Id and Binding.MappingId are strings on the wire, as are the ids in the capture/mapping/binding requests. RecordDispatchRequest.task_id is deliberately unchanged. What the collections buy, beyond storage: a payload nobody has modelled yet is inspectable, a mapping is editable at runtime, and a binding's relation to its mapping is enforced. The old schema declared no FOREIGN KEY on that link, so a dangling mapping_id was previously possible and is now refused. Behaviour preserved deliberately, each recovered from the SQLite version rather than reimplemented from memory: - draft -> pending_approval -> armed, with approval only from pending_approval and an edit always returning a binding for re-approval - one binding per source, guarded on insert, update and approve - an empty secret on update keeps the stored one, so an edit form that does not echo it back cannot silently disarm a binding - ErrMappingNotFound / ErrBindingNotFound on a missing delete target - sentinels are re-exported from storecontract, never redefined: staterpc matches on exact canonical message to rehydrate them, so a second definition would break errors.Is across the wire invisibly Two implementation notes worth keeping: - at-most-once dispatch is now a unique index rather than INSERT OR IGNORE. Only the unique-violation maps to ErrAlreadyDispatched; any other save error is returned, because treating them all as "already recorded" would turn the ledger into a silent at-least-once dispatcher. - captures store received_at themselves at nanosecond precision in a fixed-width layout. PocketBase autodates are millisecond and record ids are random, so same-millisecond captures would otherwise order arbitrarily; the old store got insertion order free from autoincrement. - ListUndispatchedCaptures is raw SQL: the anti-join is a subquery and PocketBase's filter DSL is not SQL. A read fires no hooks either way. Adds tool_calls, which had no persistence at all: tool calls only ever reached the on-disk task log, readable as text but never queryable or watchable. Nothing writes it yet. The binding secret cipher moves with the bindings it protects. --- cmd/archie-state-store/main_test.go | 14 +- cmd/archie-ui/main_test.go | 7 +- .../app/agentworker/task_execution_test.go | 6 +- internal/app/archied/binding_cipher.go | 8 +- internal/app/archied/bootstrap.go | 4 +- internal/app/archied/state_store.go | 47 +- .../app/archied/state_store_consumers_test.go | 24 +- internal/app/archied/state_store_test.go | 9 +- internal/app/archieui/degraded_routes_test.go | 4 +- internal/app/archieui/run_test.go | 4 +- internal/app/archieui/task_logs_test.go | 4 +- internal/contracts/state/v1/state.pb.go | 135 ++--- internal/daemon/daemon.go | 4 +- internal/daemon/dispatch_bindings_test.go | 87 +-- internal/domain/binding/binding.go | 8 +- internal/domain/binding/binding_test.go | 2 +- internal/domain/mapping/mapping.go | 2 +- .../domain/storecontract/storecontract.go | 26 +- internal/domain/workflow/task/task.go | 4 +- .../captureintake/receiver_test.go | 28 +- .../edastore/cipher.go} | 2 +- .../edastore/cipher_test.go} | 210 ++++--- internal/infrastructure/edastore/contracts.go | 190 ++++++ .../edastore/contracts_binding.go | 370 ++++++++++++ internal/infrastructure/edastore/edastore.go | 36 +- internal/infrastructure/edastore/opentest.go | 23 + internal/infrastructure/edastore/records.go | 211 ------- internal/infrastructure/edastore/schema.go | 21 +- .../infrastructure/edastore/schema_test.go | 123 +++- .../edastore/testsupport_test.go | 30 + .../infrastructure/stateadmin/stateadmin.go | 6 +- .../staterpc/attribution_test.go | 2 +- .../staterpc/capture_stream_test.go | 28 +- internal/infrastructure/staterpc/client.go | 26 +- .../staterpc/conformance_test.go | 97 ++-- .../staterpc/task_grants_test.go | 27 +- .../staterpc/task_log_nilslice_test.go | 2 +- .../infrastructure/staterpc/task_log_test.go | 2 +- internal/store/bindings.go | 450 --------------- internal/store/bindings_test.go | 544 ------------------ internal/store/captures.go | 124 ---- internal/store/captures_test.go | 169 ------ internal/store/interface.go | 18 - internal/store/mappings.go | 162 ------ internal/store/mappings_test.go | 145 ----- internal/store/playbook_dispatches.go | 62 -- internal/store/playbook_dispatches_test.go | 73 --- internal/store/store.go | 41 +- internal/store/store_test.go | 8 +- internal/webui/api_binding.go | 19 +- internal/webui/api_binding_test.go | 45 +- internal/webui/api_capture_test.go | 11 +- internal/webui/api_mapping.go | 17 +- internal/webui/api_mapping_test.go | 23 +- internal/webui/remote_store_test.go | 17 +- proto/state/v1/state.proto | 33 +- 56 files changed, 1332 insertions(+), 2462 deletions(-) rename internal/{store/binding_cipher.go => infrastructure/edastore/cipher.go} (99%) rename internal/{store/binding_cipher_test.go => infrastructure/edastore/cipher_test.go} (58%) create mode 100644 internal/infrastructure/edastore/contracts.go create mode 100644 internal/infrastructure/edastore/contracts_binding.go create mode 100644 internal/infrastructure/edastore/opentest.go delete mode 100644 internal/infrastructure/edastore/records.go create mode 100644 internal/infrastructure/edastore/testsupport_test.go delete mode 100644 internal/store/bindings.go delete mode 100644 internal/store/bindings_test.go delete mode 100644 internal/store/captures.go delete mode 100644 internal/store/captures_test.go delete mode 100644 internal/store/mappings.go delete mode 100644 internal/store/mappings_test.go delete mode 100644 internal/store/playbook_dispatches.go delete mode 100644 internal/store/playbook_dispatches_test.go diff --git a/cmd/archie-state-store/main_test.go b/cmd/archie-state-store/main_test.go index 6546c38c0..a11b0c99d 100644 --- a/cmd/archie-state-store/main_test.go +++ b/cmd/archie-state-store/main_test.go @@ -372,8 +372,8 @@ func TestStateStoreRealProcessSmoke(t *testing.T) { }); err != nil { t.Fatalf("InsertEvent config_captured: %v", err) } - if capID, err := cl.InsertCapture(ctx, capture("sentry", `{"id":1}`), 0, 0); err != nil || capID == 0 { - t.Fatalf("InsertCapture = (%d, %v)", capID, err) + if capID, err := cl.InsertCapture(ctx, capture("sentry", `{"id":1}`), 0, 0); err != nil || capID == "" { + t.Fatalf("InsertCapture = (%q, %v)", capID, err) } got, err := cl.TaskByID(ctx, task.ID) @@ -583,8 +583,8 @@ func TestStateStoreRealProcessRemoteSurfaces(t *testing.T) { ctx := t.Context() mappingID, err := cl.InsertMapping(ctx, mapping.Mapping{Name: "m1", Fields: []mapping.Field{{Name: "a", Path: "a", Type: mapping.TypeString}}}) - if err != nil || mappingID == 0 { - t.Fatalf("InsertMapping = (%d, %v)", mappingID, err) + if err != nil || mappingID == "" { + t.Fatalf("InsertMapping = (%q, %v)", mappingID, err) } if got, err := cl.GetMapping(ctx, mappingID); err != nil || got == nil || got.Name != "m1" { t.Fatalf("GetMapping = %+v, %v", got, err) @@ -593,8 +593,8 @@ func TestStateStoreRealProcessRemoteSurfaces(t *testing.T) { Name: "b1", Matcher: binding.Matcher{Source: "sentry"}, MappingID: mappingID, Workflow: "implement", Secret: "0123456789abcdef0123456789abcdef", }) - if err != nil || bindingID == 0 { - t.Fatalf("InsertBinding = (%d, %v)", bindingID, err) + if err != nil || bindingID == "" { + t.Fatalf("InsertBinding = (%q, %v)", bindingID, err) } // A binding starts as draft and must be edited (draft -> pending_approval) // then approved (pending_approval -> armed) before it is armed for a @@ -613,7 +613,7 @@ func TestStateStoreRealProcessRemoteSurfaces(t *testing.T) { t.Fatalf("ArmedBindingsForSource = %+v, %v", armed, err) } // Remote surfaces share error-sentinel fidelity on the wire too. - missing := mapping.Mapping{ID: mappingID + 999999, Name: "x", Fields: []mapping.Field{{Name: "a", Path: "a", Type: mapping.TypeString}}} + missing := mapping.Mapping{ID: "rabsent00000000", Name: "x", Fields: []mapping.Field{{Name: "a", Path: "a", Type: mapping.TypeString}}} if err := cl.UpdateMapping(ctx, missing); !errors.Is(err, store.ErrMappingNotFound) { t.Fatalf("UpdateMapping missing = %v, want ErrMappingNotFound", err) } diff --git a/cmd/archie-ui/main_test.go b/cmd/archie-ui/main_test.go index fc8fc2cee..60a902064 100644 --- a/cmd/archie-ui/main_test.go +++ b/cmd/archie-ui/main_test.go @@ -37,6 +37,7 @@ import ( "github.com/samcharles93/archie-core/internal/domain/health" "github.com/samcharles93/archie-core/internal/domain/messaging" "github.com/samcharles93/archie-core/internal/domain/taskactions" + "github.com/samcharles93/archie-core/internal/infrastructure/edastore" "github.com/samcharles93/archie-core/internal/infrastructure/gatewayrpc" "github.com/samcharles93/archie-core/internal/infrastructure/staterpc" "github.com/samcharles93/archie-core/internal/store" @@ -601,8 +602,9 @@ func TestUIProcessLinksMultiIdentityTaskRowsToTheirOwningForge(t *testing.T) { } stateTarget, stopState := serveGRPC(t, func(r grpc.ServiceRegistrar) { + eda := edastore.OpenTest(t) staterpc.RegisterServer(r, staterpc.Deps{ - Tasks: st, Captures: st, BindingDispatcher: st, ConfigSnapshots: st, + Tasks: st, Captures: eda, BindingDispatcher: eda, ConfigSnapshots: st, Log: slog.New(slog.DiscardHandler), }) }) @@ -657,8 +659,9 @@ func TestUIProcessServesTheDashboardAgainstLiveDependencies(t *testing.T) { gateway := &fakeGateway{} stateTarget, stopState := serveGRPC(t, func(r grpc.ServiceRegistrar) { + eda := edastore.OpenTest(t) staterpc.RegisterServer(r, staterpc.Deps{ - Tasks: st, Captures: st, BindingDispatcher: st, ConfigSnapshots: st, + Tasks: st, Captures: eda, BindingDispatcher: eda, ConfigSnapshots: st, Log: slog.New(slog.DiscardHandler), }) }) diff --git a/internal/app/agentworker/task_execution_test.go b/internal/app/agentworker/task_execution_test.go index a4933d388..208e6e043 100644 --- a/internal/app/agentworker/task_execution_test.go +++ b/internal/app/agentworker/task_execution_test.go @@ -30,6 +30,7 @@ import ( "github.com/samcharles93/archie-core/internal/forge" "github.com/samcharles93/archie-core/internal/forgerpc" agentnats "github.com/samcharles93/archie-core/internal/infrastructure/agenttransport/nats" + "github.com/samcharles93/archie-core/internal/infrastructure/edastore" "github.com/samcharles93/archie-core/internal/infrastructure/staterpc" "github.com/samcharles93/archie-core/internal/infrastructure/workflowsteps" "github.com/samcharles93/archie-core/internal/installtype" @@ -376,9 +377,10 @@ func startStateStoreGRPC(t *testing.T, local *store.Store) string { t.Fatal(err) } server := grpc.NewServer() + eda := edastore.OpenTest(t) staterpc.RegisterServer(server, staterpc.Deps{ - Tasks: local, Captures: local, Mappings: local, Bindings: local, - BindingDispatcher: local, BindingTaskCreator: local, + Tasks: local, Captures: eda, Mappings: eda, Bindings: eda, + BindingDispatcher: eda, BindingTaskCreator: local, }) go func() { _ = server.Serve(listener) }() t.Cleanup(func() { server.Stop(); _ = listener.Close() }) diff --git a/internal/app/archied/binding_cipher.go b/internal/app/archied/binding_cipher.go index b5fc1f465..bbec81d1a 100644 --- a/internal/app/archied/binding_cipher.go +++ b/internal/app/archied/binding_cipher.go @@ -5,15 +5,15 @@ import ( "strings" "github.com/samcharles93/archie-core/internal/config" + "github.com/samcharles93/archie-core/internal/infrastructure/edastore" "github.com/samcharles93/archie-core/internal/secret" - "github.com/samcharles93/archie-core/internal/store" ) // bindingCipherFromConfig resolves the binding at-rest encryption key from -// config through the secret registry and returns a store cipher. An unset +// config through the secret registry and returns the EDA store's cipher. An unset // encryption_key returns a nil cipher, keeping the legacy plaintext path (the // store's default). See docs/prds/binding-secret-encryption.md. -func bindingCipherFromConfig(cfg config.Config, secrets *secret.Registry) (store.BindingCipher, error) { +func bindingCipherFromConfig(cfg config.Config, secrets *secret.Registry) (edastore.BindingCipher, error) { if cfg.Bindings.EncryptionKey == (secret.SecretRef{}) { return nil, nil } @@ -36,5 +36,5 @@ func bindingCipherFromConfig(cfg config.Config, secrets *secret.Registry) (store previous = append(previous, v) } } - return store.NewBindingCipher(active, previous) + return edastore.NewBindingCipher(active, previous) } diff --git a/internal/app/archied/bootstrap.go b/internal/app/archied/bootstrap.go index 72e526297..9480f9269 100644 --- a/internal/app/archied/bootstrap.go +++ b/internal/app/archied/bootstrap.go @@ -46,6 +46,7 @@ import ( forgewebhook "github.com/samcharles93/archie-core/internal/forge/webhook" "github.com/samcharles93/archie-core/internal/gateway" "github.com/samcharles93/archie-core/internal/infrastructure/configuration" + "github.com/samcharles93/archie-core/internal/infrastructure/edastore" infraembedding "github.com/samcharles93/archie-core/internal/infrastructure/embedding" "github.com/samcharles93/archie-core/internal/infrastructure/eventbus/nats" infraMemory "github.com/samcharles93/archie-core/internal/infrastructure/memory" @@ -105,7 +106,8 @@ type boot struct { playbooks *playbook.Store - st storecontract.TaskStore + st storecontract.TaskStore + eda *edastore.Store // stateStore is the State Store contract adapter every daemon and gateway // store consumer depends on. It is ALWAYS the remote *staterpc.Client // dialed to the standalone archie-state-store gRPC service at diff --git a/internal/app/archied/state_store.go b/internal/app/archied/state_store.go index 7b6ca1498..1756b2506 100644 --- a/internal/app/archied/state_store.go +++ b/internal/app/archied/state_store.go @@ -25,6 +25,7 @@ import ( "github.com/samcharles93/archie-core/internal/domain/health" "github.com/samcharles93/archie-core/internal/domain/identity" "github.com/samcharles93/archie-core/internal/domain/storecontract" + "github.com/samcharles93/archie-core/internal/infrastructure/edastore" "github.com/samcharles93/archie-core/internal/infrastructure/readiness" "github.com/samcharles93/archie-core/internal/infrastructure/stateadmin" "github.com/samcharles93/archie-core/internal/infrastructure/staterpc" @@ -46,6 +47,11 @@ func configuredIdentityNames(cfg config.Config) []string { return names } +// edaDBPath derives the event-capture database from the configured db_path, +// beside the task database rather than inside it: the two stores have separate +// lifecycles and only meet at binding_dispatches.task_id. +func edaDBPath(configuredPath string) string { return configuredPath + "-eda.sqlite" } + // StateStoreOptions contains process inputs for the standalone State Store. type StateStoreOptions struct { Config string @@ -221,7 +227,23 @@ func (b *boot) openStateStore(ctx context.Context) error { log.Error("release state store ownership", "err", err) } }) - st, err := openProductionTaskStore(ctx, path, store.WithBindingCipher(bindingCipher)) + eda, err := edastore.Open(edastore.Config{ + DBPath: edaDBPath(cfg.DBPath), + DataDir: filepath.Join(filepath.Dir(path), "eda"), + Cipher: bindingCipher, + }) + if err != nil { + log.Error("open event-capture store", "err", err) + return err + } + b.eda = eda + b.addCleanup(func() { + if err := eda.Close(); err != nil { + log.Error("close event-capture store", "err", err) + } + }) + + st, err := openProductionTaskStore(ctx, path) if err != nil { log.Error("open state store", "err", err) return err @@ -268,20 +290,15 @@ func (b *boot) stateStoreDeps(grants *staterpc.TaskGrants) staterpc.Deps { if b.taskLogs != nil { deps.TaskLogs = b.taskLogs } - if cs, ok := b.st.(storecontract.CaptureStore); ok { - deps.Captures = cs - } - if ms, ok := b.st.(storecontract.MappingStore); ok { - deps.Mappings = ms - } - if bs, ok := b.st.(storecontract.BindingStore); ok { - deps.Bindings = bs - } - if bd, ok := b.st.(storecontract.BindingDispatcher); ok { - deps.BindingDispatcher = bd - } - if pd, ok := b.st.(storecontract.PlaybookDispatcher); ok { - deps.PlaybookDispatcher = pd + // The event-capture contracts are served by the PocketBase store, not the + // task store: those tables moved. BindingTaskCreator stays below on the + // task store, because creating a task is the one thing it still does. + if b.eda != nil { + deps.Captures = b.eda + deps.Mappings = b.eda + deps.Bindings = b.eda + deps.BindingDispatcher = b.eda + deps.PlaybookDispatcher = b.eda } if btc, ok := b.st.(storecontract.BindingTaskCreator); ok { deps.BindingTaskCreator = btc diff --git a/internal/app/archied/state_store_consumers_test.go b/internal/app/archied/state_store_consumers_test.go index bc2e8f408..a0cbb6c3e 100644 --- a/internal/app/archied/state_store_consumers_test.go +++ b/internal/app/archied/state_store_consumers_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/samcharles93/archie-core/internal/config" + "github.com/samcharles93/archie-core/internal/infrastructure/edastore" taskactionstore "github.com/samcharles93/archie-core/internal/infrastructure/taskactions" "github.com/samcharles93/archie-core/internal/store" ) @@ -15,14 +16,33 @@ import ( // b.stateStore rather than b.st: the two differ by identity, so an assertion // that a consumer field holds storeB (not storeA) can only pass if the wiring // reads b.stateStore. -func openSecondStore(t *testing.T) *store.Store { +// stateStoreAdapter stands in for the real adapter (*staterpc.Client), which +// fronts both halves of the contract over one connection: the task store and +// the event-capture store. A plain *store.Store no longer satisfies the +// event-capture surfaces, so a test that asserts "everything routes through +// the adapter" needs something that implements them, exactly as the client +// does in production. +// edaSide aliases the event-capture store so it can be embedded beside +// *store.Store: both are named Store, and Go allows only one embedded field +// per name. +type edaSide = edastore.Store + +type stateStoreAdapter struct { + *store.Store + *edaSide +} + +// Close disambiguates the two embedded stores, which both have one. +func (a *stateStoreAdapter) Close() error { return a.Store.Close() } + +func openSecondStore(t *testing.T) *stateStoreAdapter { t.Helper() st, err := store.Open(t.Context(), filepath.Join(t.TempDir(), "state-store.db")) if err != nil { t.Fatalf("open second store: %v", err) } t.Cleanup(func() { _ = st.Close() }) - return st + return &stateStoreAdapter{Store: st, edaSide: edastore.OpenTest(t)} } // TestBuildDaemonRoutesTaskStoreThroughStateStore proves the daemon's task diff --git a/internal/app/archied/state_store_test.go b/internal/app/archied/state_store_test.go index d9007b16c..363a2b9f4 100644 --- a/internal/app/archied/state_store_test.go +++ b/internal/app/archied/state_store_test.go @@ -12,6 +12,7 @@ import ( "github.com/samcharles93/archie-core/internal/config" pb "github.com/samcharles93/archie-core/internal/contracts/state/v1" "github.com/samcharles93/archie-core/internal/domain/storecontract" + "github.com/samcharles93/archie-core/internal/infrastructure/edastore" "github.com/samcharles93/archie-core/internal/infrastructure/staterpc" "github.com/samcharles93/archie-core/internal/logging" "github.com/samcharles93/archie-core/internal/store" @@ -145,12 +146,16 @@ func TestStateStoreDepsServePlaybookDispatcher(t *testing.T) { b := newBootstrap() b.st = st + // The playbook ledger moved to the event-capture store with the rest of + // the dispatch tables; the task store no longer serves it. + eda := edastore.OpenTest(t) + b.eda = eda deps := b.stateStoreDeps(&staterpc.TaskGrants{}) if deps.PlaybookDispatcher == nil { t.Fatal("stateStoreDeps leaves PlaybookDispatcher nil; the standalone State Store is the only production server for the playbook dispatch ledger") } - if deps.PlaybookDispatcher != storecontract.PlaybookDispatcher(st) { - t.Fatalf("PlaybookDispatcher = %T, want the opened *store.Store", deps.PlaybookDispatcher) + if deps.PlaybookDispatcher != storecontract.PlaybookDispatcher(eda) { + t.Fatalf("PlaybookDispatcher = %T, want the opened *edastore.Store", deps.PlaybookDispatcher) } // A boot without a store must not fabricate one: nil keeps the RPCs honest // as unavailable rather than depending on a nil receiver. diff --git a/internal/app/archieui/degraded_routes_test.go b/internal/app/archieui/degraded_routes_test.go index 91301a15d..ac245e04a 100644 --- a/internal/app/archieui/degraded_routes_test.go +++ b/internal/app/archieui/degraded_routes_test.go @@ -13,6 +13,7 @@ import ( "google.golang.org/grpc" + "github.com/samcharles93/archie-core/internal/infrastructure/edastore" "github.com/samcharles93/archie-core/internal/infrastructure/staterpc" "github.com/samcharles93/archie-core/internal/logging" "github.com/samcharles93/archie-core/internal/store" @@ -165,8 +166,9 @@ func composeUIProcess(t *testing.T) (*webui.Server, int64) { logs := logging.NewTaskRegistry(filepath.Join(t.TempDir(), "logs", "tasks"), logging.NewFeed(10), logging.TaskSinkOptions{}) target, stop := serveGRPC(t, func(r grpc.ServiceRegistrar) { + eda := edastore.OpenTest(t) staterpc.RegisterServer(r, staterpc.Deps{ - Tasks: st, Captures: st, BindingDispatcher: st, ConfigSnapshots: st, + Tasks: st, Captures: eda, BindingDispatcher: eda, ConfigSnapshots: st, TaskLogs: logs, Log: slog.New(slog.DiscardHandler), }) }) diff --git a/internal/app/archieui/run_test.go b/internal/app/archieui/run_test.go index 811b05cc7..47529eecb 100644 --- a/internal/app/archieui/run_test.go +++ b/internal/app/archieui/run_test.go @@ -21,6 +21,7 @@ import ( "github.com/samcharles93/archie-core/internal/domain/taskactions" "github.com/samcharles93/archie-core/internal/events" "github.com/samcharles93/archie-core/internal/gateway" + "github.com/samcharles93/archie-core/internal/infrastructure/edastore" "github.com/samcharles93/archie-core/internal/infrastructure/gatewayrpc" "github.com/samcharles93/archie-core/internal/infrastructure/staterpc" "github.com/samcharles93/archie-core/internal/store" @@ -151,7 +152,8 @@ func TestUIServesDashboardAgainstRemoteContracts(t *testing.T) { } stateTarget, stopState := serveGRPC(t, func(r grpc.ServiceRegistrar) { - staterpc.RegisterServer(r, staterpc.Deps{Tasks: st, Captures: st, BindingDispatcher: st, ConfigSnapshots: st, Log: slog.New(slog.DiscardHandler)}) + eda := edastore.OpenTest(t) + staterpc.RegisterServer(r, staterpc.Deps{Tasks: st, Captures: eda, BindingDispatcher: eda, ConfigSnapshots: st, Log: slog.New(slog.DiscardHandler)}) }) defer stopState() chat := &fakeChat{sessions: []gateway.SessionContext{ diff --git a/internal/app/archieui/task_logs_test.go b/internal/app/archieui/task_logs_test.go index 0dc85afb6..dffa54f2e 100644 --- a/internal/app/archieui/task_logs_test.go +++ b/internal/app/archieui/task_logs_test.go @@ -13,6 +13,7 @@ import ( "google.golang.org/grpc" + "github.com/samcharles93/archie-core/internal/infrastructure/edastore" "github.com/samcharles93/archie-core/internal/infrastructure/staterpc" "github.com/samcharles93/archie-core/internal/logging" "github.com/samcharles93/archie-core/internal/store" @@ -178,8 +179,9 @@ func composeUIProcessWithLog(t *testing.T) (*webui.Server, int64, int) { } target, stop := serveGRPC(t, func(r grpc.ServiceRegistrar) { + eda := edastore.OpenTest(t) staterpc.RegisterServer(r, staterpc.Deps{ - Tasks: st, Captures: st, BindingDispatcher: st, ConfigSnapshots: st, + Tasks: st, Captures: eda, BindingDispatcher: eda, ConfigSnapshots: st, TaskLogs: logs, Log: slog.New(slog.DiscardHandler), }) }) diff --git a/internal/contracts/state/v1/state.pb.go b/internal/contracts/state/v1/state.pb.go index f8231fd85..d96463dec 100644 --- a/internal/contracts/state/v1/state.pb.go +++ b/internal/contracts/state/v1/state.pb.go @@ -1288,7 +1288,7 @@ type Task struct { WatchCommentId int64 `protobuf:"varint,20,opt,name=watch_comment_id,json=watchCommentId,proto3" json:"watch_comment_id,omitempty"` Source string `protobuf:"bytes,21,opt,name=source,proto3" json:"source,omitempty"` Identity string `protobuf:"bytes,22,opt,name=identity,proto3" json:"identity,omitempty"` - BindingId int64 `protobuf:"varint,23,opt,name=binding_id,json=bindingId,proto3" json:"binding_id,omitempty"` + BindingId string `protobuf:"bytes,23,opt,name=binding_id,json=bindingId,proto3" json:"binding_id,omitempty"` BindingVersion int64 `protobuf:"varint,24,opt,name=binding_version,json=bindingVersion,proto3" json:"binding_version,omitempty"` CreatedAt *timestamppb.Timestamp `protobuf:"bytes,25,opt,name=created_at,json=createdAt,proto3" json:"created_at,omitempty"` UpdatedAt *timestamppb.Timestamp `protobuf:"bytes,26,opt,name=updated_at,json=updatedAt,proto3" json:"updated_at,omitempty"` @@ -1490,11 +1490,11 @@ func (x *Task) GetIdentity() string { return "" } -func (x *Task) GetBindingId() int64 { +func (x *Task) GetBindingId() string { if x != nil { return x.BindingId } - return 0 + return "" } func (x *Task) GetBindingVersion() int64 { @@ -1711,7 +1711,7 @@ func (x *Event) GetPrincipalId() string { // CapturedEvent mirrors internal/store.CapturedEvent. type CapturedEvent struct { state protoimpl.MessageState `protogen:"open.v1"` - Id int64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` ReceivedAt *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=received_at,json=receivedAt,proto3" json:"received_at,omitempty"` Source string `protobuf:"bytes,3,opt,name=source,proto3" json:"source,omitempty"` RemoteAddr string `protobuf:"bytes,4,opt,name=remote_addr,json=remoteAddr,proto3" json:"remote_addr,omitempty"` @@ -1753,11 +1753,11 @@ func (*CapturedEvent) Descriptor() ([]byte, []int) { return file_state_v1_state_proto_rawDescGZIP(), []int{26} } -func (x *CapturedEvent) GetId() int64 { +func (x *CapturedEvent) GetId() string { if x != nil { return x.Id } - return 0 + return "" } func (x *CapturedEvent) GetReceivedAt() *timestamppb.Timestamp { @@ -1881,7 +1881,7 @@ func (x *MappingField) GetRequired() bool { // Mapping mirrors internal/domain/mapping.Mapping. type Mapping struct { state protoimpl.MessageState `protogen:"open.v1"` - Id int64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` SourceHint string `protobuf:"bytes,3,opt,name=source_hint,json=sourceHint,proto3" json:"source_hint,omitempty"` Fields []*MappingField `protobuf:"bytes,4,rep,name=fields,proto3" json:"fields,omitempty"` @@ -1921,11 +1921,11 @@ func (*Mapping) Descriptor() ([]byte, []int) { return file_state_v1_state_proto_rawDescGZIP(), []int{28} } -func (x *Mapping) GetId() int64 { +func (x *Mapping) GetId() string { if x != nil { return x.Id } - return 0 + return "" } func (x *Mapping) GetName() string { @@ -2011,10 +2011,10 @@ func (x *BindingMatcher) GetSource() string { // Binding mirrors internal/domain/binding.Binding. type Binding struct { state protoimpl.MessageState `protogen:"open.v1"` - Id int64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` Matcher *BindingMatcher `protobuf:"bytes,3,opt,name=matcher,proto3" json:"matcher,omitempty"` - MappingId int64 `protobuf:"varint,4,opt,name=mapping_id,json=mappingId,proto3" json:"mapping_id,omitempty"` + MappingId string `protobuf:"bytes,4,opt,name=mapping_id,json=mappingId,proto3" json:"mapping_id,omitempty"` Workflow string `protobuf:"bytes,5,opt,name=workflow,proto3" json:"workflow,omitempty"` Owner string `protobuf:"bytes,6,opt,name=owner,proto3" json:"owner,omitempty"` Repo string `protobuf:"bytes,7,opt,name=repo,proto3" json:"repo,omitempty"` @@ -2057,11 +2057,11 @@ func (*Binding) Descriptor() ([]byte, []int) { return file_state_v1_state_proto_rawDescGZIP(), []int{30} } -func (x *Binding) GetId() int64 { +func (x *Binding) GetId() string { if x != nil { return x.Id } - return 0 + return "" } func (x *Binding) GetName() string { @@ -2078,11 +2078,11 @@ func (x *Binding) GetMatcher() *BindingMatcher { return nil } -func (x *Binding) GetMappingId() int64 { +func (x *Binding) GetMappingId() string { if x != nil { return x.MappingId } - return 0 + return "" } func (x *Binding) GetWorkflow() string { @@ -5763,7 +5763,7 @@ func (x *InsertCaptureRequest) GetMaxEvents() int64 { type InsertCaptureResponse struct { state protoimpl.MessageState `protogen:"open.v1"` - Id int64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -5798,11 +5798,11 @@ func (*InsertCaptureResponse) Descriptor() ([]byte, []int) { return file_state_v1_state_proto_rawDescGZIP(), []int{101} } -func (x *InsertCaptureResponse) GetId() int64 { +func (x *InsertCaptureResponse) GetId() string { if x != nil { return x.Id } - return 0 + return "" } type ListCapturesRequest struct { @@ -6028,7 +6028,7 @@ func (x *InsertMappingRequest) GetMapping() *Mapping { type InsertMappingResponse struct { state protoimpl.MessageState `protogen:"open.v1"` - Id int64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -6063,16 +6063,16 @@ func (*InsertMappingResponse) Descriptor() ([]byte, []int) { return file_state_v1_state_proto_rawDescGZIP(), []int{107} } -func (x *InsertMappingResponse) GetId() int64 { +func (x *InsertMappingResponse) GetId() string { if x != nil { return x.Id } - return 0 + return "" } type GetMappingRequest struct { state protoimpl.MessageState `protogen:"open.v1"` - Id int64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -6107,11 +6107,11 @@ func (*GetMappingRequest) Descriptor() ([]byte, []int) { return file_state_v1_state_proto_rawDescGZIP(), []int{108} } -func (x *GetMappingRequest) GetId() int64 { +func (x *GetMappingRequest) GetId() string { if x != nil { return x.Id } - return 0 + return "" } type GetMappingResponse struct { @@ -6328,7 +6328,7 @@ func (*UpdateMappingResponse) Descriptor() ([]byte, []int) { type DeleteMappingRequest struct { state protoimpl.MessageState `protogen:"open.v1"` - Id int64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -6363,11 +6363,11 @@ func (*DeleteMappingRequest) Descriptor() ([]byte, []int) { return file_state_v1_state_proto_rawDescGZIP(), []int{114} } -func (x *DeleteMappingRequest) GetId() int64 { +func (x *DeleteMappingRequest) GetId() string { if x != nil { return x.Id } - return 0 + return "" } type DeleteMappingResponse struct { @@ -6452,7 +6452,7 @@ func (x *InsertBindingRequest) GetBinding() *Binding { type InsertBindingResponse struct { state protoimpl.MessageState `protogen:"open.v1"` - Id int64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -6487,16 +6487,16 @@ func (*InsertBindingResponse) Descriptor() ([]byte, []int) { return file_state_v1_state_proto_rawDescGZIP(), []int{117} } -func (x *InsertBindingResponse) GetId() int64 { +func (x *InsertBindingResponse) GetId() string { if x != nil { return x.Id } - return 0 + return "" } type GetBindingRequest struct { state protoimpl.MessageState `protogen:"open.v1"` - Id int64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -6531,11 +6531,11 @@ func (*GetBindingRequest) Descriptor() ([]byte, []int) { return file_state_v1_state_proto_rawDescGZIP(), []int{118} } -func (x *GetBindingRequest) GetId() int64 { +func (x *GetBindingRequest) GetId() string { if x != nil { return x.Id } - return 0 + return "" } type GetBindingResponse struct { @@ -6752,7 +6752,7 @@ func (*UpdateBindingResponse) Descriptor() ([]byte, []int) { type DeleteBindingRequest struct { state protoimpl.MessageState `protogen:"open.v1"` - Id int64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -6787,11 +6787,11 @@ func (*DeleteBindingRequest) Descriptor() ([]byte, []int) { return file_state_v1_state_proto_rawDescGZIP(), []int{124} } -func (x *DeleteBindingRequest) GetId() int64 { +func (x *DeleteBindingRequest) GetId() string { if x != nil { return x.Id } - return 0 + return "" } type DeleteBindingResponse struct { @@ -6832,7 +6832,7 @@ func (*DeleteBindingResponse) Descriptor() ([]byte, []int) { type ApproveBindingRequest struct { state protoimpl.MessageState `protogen:"open.v1"` - Id int64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -6867,11 +6867,11 @@ func (*ApproveBindingRequest) Descriptor() ([]byte, []int) { return file_state_v1_state_proto_rawDescGZIP(), []int{126} } -func (x *ApproveBindingRequest) GetId() int64 { +func (x *ApproveBindingRequest) GetId() string { if x != nil { return x.Id } - return 0 + return "" } type ApproveBindingResponse struct { @@ -7000,12 +7000,13 @@ func (x *ArmedBindingsForSourceResponse) GetBindings() []*Binding { type RecordDispatchRequest struct { state protoimpl.MessageState `protogen:"open.v1"` - BindingId int64 `protobuf:"varint,1,opt,name=binding_id,json=bindingId,proto3" json:"binding_id,omitempty"` + BindingId string `protobuf:"bytes,1,opt,name=binding_id,json=bindingId,proto3" json:"binding_id,omitempty"` BindingVersion int64 `protobuf:"varint,2,opt,name=binding_version,json=bindingVersion,proto3" json:"binding_version,omitempty"` - CaptureId int64 `protobuf:"varint,3,opt,name=capture_id,json=captureId,proto3" json:"capture_id,omitempty"` - TaskId int64 `protobuf:"varint,4,opt,name=task_id,json=taskId,proto3" json:"task_id,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + CaptureId string `protobuf:"bytes,3,opt,name=capture_id,json=captureId,proto3" json:"capture_id,omitempty"` + // task_id addresses the SQLite-owned task tables, which are not migrating. + TaskId int64 `protobuf:"varint,4,opt,name=task_id,json=taskId,proto3" json:"task_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *RecordDispatchRequest) Reset() { @@ -7038,11 +7039,11 @@ func (*RecordDispatchRequest) Descriptor() ([]byte, []int) { return file_state_v1_state_proto_rawDescGZIP(), []int{130} } -func (x *RecordDispatchRequest) GetBindingId() int64 { +func (x *RecordDispatchRequest) GetBindingId() string { if x != nil { return x.BindingId } - return 0 + return "" } func (x *RecordDispatchRequest) GetBindingVersion() int64 { @@ -7052,11 +7053,11 @@ func (x *RecordDispatchRequest) GetBindingVersion() int64 { return 0 } -func (x *RecordDispatchRequest) GetCaptureId() int64 { +func (x *RecordDispatchRequest) GetCaptureId() string { if x != nil { return x.CaptureId } - return 0 + return "" } func (x *RecordDispatchRequest) GetTaskId() int64 { @@ -7487,7 +7488,7 @@ type EnqueueBindingTaskRequest struct { Body string `protobuf:"bytes,4,opt,name=body,proto3" json:"body,omitempty"` Workflow string `protobuf:"bytes,5,opt,name=workflow,proto3" json:"workflow,omitempty"` Identity string `protobuf:"bytes,6,opt,name=identity,proto3" json:"identity,omitempty"` - BindingId int64 `protobuf:"varint,7,opt,name=binding_id,json=bindingId,proto3" json:"binding_id,omitempty"` + BindingId string `protobuf:"bytes,7,opt,name=binding_id,json=bindingId,proto3" json:"binding_id,omitempty"` BindingVersion int64 `protobuf:"varint,8,opt,name=binding_version,json=bindingVersion,proto3" json:"binding_version,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -7565,11 +7566,11 @@ func (x *EnqueueBindingTaskRequest) GetIdentity() string { return "" } -func (x *EnqueueBindingTaskRequest) GetBindingId() int64 { +func (x *EnqueueBindingTaskRequest) GetBindingId() string { if x != nil { return x.BindingId } - return 0 + return "" } func (x *EnqueueBindingTaskRequest) GetBindingVersion() int64 { @@ -7734,7 +7735,7 @@ const file_state_v1_state_proto_rawDesc = "" + "\x06source\x18\x15 \x01(\tR\x06source\x12\x1a\n" + "\bidentity\x18\x16 \x01(\tR\bidentity\x12\x1d\n" + "\n" + - "binding_id\x18\x17 \x01(\x03R\tbindingId\x12'\n" + + "binding_id\x18\x17 \x01(\tR\tbindingId\x12'\n" + "\x0fbinding_version\x18\x18 \x01(\x03R\x0ebindingVersion\x129\n" + "\n" + "created_at\x18\x19 \x01(\v2\x1a.google.protobuf.TimestampR\tcreatedAt\x129\n" + @@ -7762,7 +7763,7 @@ const file_state_v1_state_proto_rawDesc = "" + "actor_kind\x18\r \x01(\tR\tactorKind\x12!\n" + "\fprincipal_id\x18\x0e \x01(\tR\vprincipalId\"\x8c\x02\n" + "\rCapturedEvent\x12\x0e\n" + - "\x02id\x18\x01 \x01(\x03R\x02id\x12;\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12;\n" + "\vreceived_at\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampR\n" + "receivedAt\x12\x16\n" + "\x06source\x18\x03 \x01(\tR\x06source\x12\x1f\n" + @@ -7778,7 +7779,7 @@ const file_state_v1_state_proto_rawDesc = "" + "\x04type\x18\x03 \x01(\tR\x04type\x12\x1a\n" + "\brequired\x18\x04 \x01(\bR\brequired\"\xf4\x01\n" + "\aMapping\x12\x0e\n" + - "\x02id\x18\x01 \x01(\x03R\x02id\x12\x12\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12\x12\n" + "\x04name\x18\x02 \x01(\tR\x04name\x12\x1f\n" + "\vsource_hint\x18\x03 \x01(\tR\n" + "sourceHint\x12.\n" + @@ -7790,11 +7791,11 @@ const file_state_v1_state_proto_rawDesc = "" + "\x0eBindingMatcher\x12\x16\n" + "\x06source\x18\x01 \x01(\tR\x06source\"\x86\x03\n" + "\aBinding\x12\x0e\n" + - "\x02id\x18\x01 \x01(\x03R\x02id\x12\x12\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12\x12\n" + "\x04name\x18\x02 \x01(\tR\x04name\x122\n" + "\amatcher\x18\x03 \x01(\v2\x18.state.v1.BindingMatcherR\amatcher\x12\x1d\n" + "\n" + - "mapping_id\x18\x04 \x01(\x03R\tmappingId\x12\x1a\n" + + "mapping_id\x18\x04 \x01(\tR\tmappingId\x12\x1a\n" + "\bworkflow\x18\x05 \x01(\tR\bworkflow\x12\x14\n" + "\x05owner\x18\x06 \x01(\tR\x05owner\x12\x12\n" + "\x04repo\x18\a \x01(\tR\x04repo\x12\x18\n" + @@ -8024,7 +8025,7 @@ const file_state_v1_state_proto_rawDesc = "" + "\n" + "max_events\x18\x03 \x01(\x03R\tmaxEvents\"'\n" + "\x15InsertCaptureResponse\x12\x0e\n" + - "\x02id\x18\x01 \x01(\x03R\x02id\"+\n" + + "\x02id\x18\x01 \x01(\tR\x02id\"+\n" + "\x13ListCapturesRequest\x12\x14\n" + "\x05limit\x18\x01 \x01(\x03R\x05limit\"K\n" + "\x14ListCapturesResponse\x123\n" + @@ -8036,9 +8037,9 @@ const file_state_v1_state_proto_rawDesc = "" + "\x14InsertMappingRequest\x12+\n" + "\amapping\x18\x01 \x01(\v2\x11.state.v1.MappingR\amapping\"'\n" + "\x15InsertMappingResponse\x12\x0e\n" + - "\x02id\x18\x01 \x01(\x03R\x02id\"#\n" + + "\x02id\x18\x01 \x01(\tR\x02id\"#\n" + "\x11GetMappingRequest\x12\x0e\n" + - "\x02id\x18\x01 \x01(\x03R\x02id\"W\n" + + "\x02id\x18\x01 \x01(\tR\x02id\"W\n" + "\x12GetMappingResponse\x12+\n" + "\amapping\x18\x01 \x01(\v2\x11.state.v1.MappingR\amapping\x12\x14\n" + "\x05found\x18\x02 \x01(\bR\x05found\"\x15\n" + @@ -8049,14 +8050,14 @@ const file_state_v1_state_proto_rawDesc = "" + "\amapping\x18\x01 \x01(\v2\x11.state.v1.MappingR\amapping\"\x17\n" + "\x15UpdateMappingResponse\"&\n" + "\x14DeleteMappingRequest\x12\x0e\n" + - "\x02id\x18\x01 \x01(\x03R\x02id\"\x17\n" + + "\x02id\x18\x01 \x01(\tR\x02id\"\x17\n" + "\x15DeleteMappingResponse\"C\n" + "\x14InsertBindingRequest\x12+\n" + "\abinding\x18\x01 \x01(\v2\x11.state.v1.BindingR\abinding\"'\n" + "\x15InsertBindingResponse\x12\x0e\n" + - "\x02id\x18\x01 \x01(\x03R\x02id\"#\n" + + "\x02id\x18\x01 \x01(\tR\x02id\"#\n" + "\x11GetBindingRequest\x12\x0e\n" + - "\x02id\x18\x01 \x01(\x03R\x02id\"W\n" + + "\x02id\x18\x01 \x01(\tR\x02id\"W\n" + "\x12GetBindingResponse\x12+\n" + "\abinding\x18\x01 \x01(\v2\x11.state.v1.BindingR\abinding\x12\x14\n" + "\x05found\x18\x02 \x01(\bR\x05found\"\x15\n" + @@ -8067,10 +8068,10 @@ const file_state_v1_state_proto_rawDesc = "" + "\abinding\x18\x01 \x01(\v2\x11.state.v1.BindingR\abinding\"\x17\n" + "\x15UpdateBindingResponse\"&\n" + "\x14DeleteBindingRequest\x12\x0e\n" + - "\x02id\x18\x01 \x01(\x03R\x02id\"\x17\n" + + "\x02id\x18\x01 \x01(\tR\x02id\"\x17\n" + "\x15DeleteBindingResponse\"'\n" + "\x15ApproveBindingRequest\x12\x0e\n" + - "\x02id\x18\x01 \x01(\x03R\x02id\"\x18\n" + + "\x02id\x18\x01 \x01(\tR\x02id\"\x18\n" + "\x16ApproveBindingResponse\"7\n" + "\x1dArmedBindingsForSourceRequest\x12\x16\n" + "\x06source\x18\x01 \x01(\tR\x06source\"O\n" + @@ -8078,10 +8079,10 @@ const file_state_v1_state_proto_rawDesc = "" + "\bbindings\x18\x01 \x03(\v2\x11.state.v1.BindingR\bbindings\"\x97\x01\n" + "\x15RecordDispatchRequest\x12\x1d\n" + "\n" + - "binding_id\x18\x01 \x01(\x03R\tbindingId\x12'\n" + + "binding_id\x18\x01 \x01(\tR\tbindingId\x12'\n" + "\x0fbinding_version\x18\x02 \x01(\x03R\x0ebindingVersion\x12\x1d\n" + "\n" + - "capture_id\x18\x03 \x01(\x03R\tcaptureId\x12\x17\n" + + "capture_id\x18\x03 \x01(\tR\tcaptureId\x12\x17\n" + "\atask_id\x18\x04 \x01(\x03R\x06taskId\"\x18\n" + "\x16RecordDispatchResponse\"\xa3\x01\n" + "\x1dRecordPlaybookDispatchRequest\x12\x1f\n" + @@ -8113,7 +8114,7 @@ const file_state_v1_state_proto_rawDesc = "" + "\bworkflow\x18\x05 \x01(\tR\bworkflow\x12\x1a\n" + "\bidentity\x18\x06 \x01(\tR\bidentity\x12\x1d\n" + "\n" + - "binding_id\x18\a \x01(\x03R\tbindingId\x12'\n" + + "binding_id\x18\a \x01(\tR\tbindingId\x12'\n" + "\x0fbinding_version\x18\b \x01(\x03R\x0ebindingVersion\"@\n" + "\x1aEnqueueBindingTaskResponse\x12\"\n" + "\x04task\x18\x01 \x01(\v2\x0e.state.v1.TaskR\x04task2\xe1)\n" + diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index d78bd8a85..7add5e30e 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -678,7 +678,7 @@ func (d *Daemon) dispatchOneBinding(ctx context.Context, b binding.Binding, c st if hasBlockingFailure(m.Fields, failures) { _, _ = d.Store.InsertEvent(ctx, events.Event{ Kind: "binding_dispatch_failure", - Detail: fmt.Sprintf("binding %d (capture %d): required field failed", b.ID, c.ID), + Detail: fmt.Sprintf("binding %q (capture %q): required field failed", b.ID, c.ID), Data: map[string]any{ "binding_id": b.ID, "binding_version": b.Version, @@ -746,7 +746,7 @@ func hasBlockingFailure(fields []mapping.Field, failures []mapping.Failure) bool // provenance to hand back to the operator. func renderBindingBody(values map[string]any, c storecontract.CapturedEvent) string { if len(values) == 0 { - return fmt.Sprintf("(no fields resolved from capture %d)", c.ID) + return fmt.Sprintf("(no fields resolved from capture %q)", c.ID) } parts := make([]string, 0, len(values)) for k, v := range values { diff --git a/internal/daemon/dispatch_bindings_test.go b/internal/daemon/dispatch_bindings_test.go index e55951eaa..63e16501b 100644 --- a/internal/daemon/dispatch_bindings_test.go +++ b/internal/daemon/dispatch_bindings_test.go @@ -11,6 +11,7 @@ import ( "github.com/samcharles93/archie-core/internal/domain/binding" "github.com/samcharles93/archie-core/internal/domain/mapping" "github.com/samcharles93/archie-core/internal/domain/workflow" + "github.com/samcharles93/archie-core/internal/infrastructure/edastore" "github.com/samcharles93/archie-core/internal/store" ) @@ -35,7 +36,7 @@ func TestDispatchBindingsCreatesTaskFromArmedCapture(t *testing.T) { } got := tasks[0] if got.BindingID != bindingID { - t.Errorf("task.BindingID = %d, want %d", got.BindingID, bindingID) + t.Errorf("task.BindingID = %q, want %q", got.BindingID, bindingID) } if got.BindingVersion != bindingVersion { t.Errorf("task.BindingVersion = %d, want %d", got.BindingVersion, bindingVersion) @@ -230,9 +231,9 @@ func TestDispatchBindingsNilDispatcherIsNoOp(t *testing.T) { d := &Daemon{ Cfg: config.NewHolder(config.Config{Repos: []config.Repo{{Owner: "acme", Name: "widget"}}}), - Store: s, - Mappings: s, - Bindings: s, + Store: s.Store, + Mappings: s.EdaStore, + Bindings: s.EdaStore, // BindingDispatcher and BindingTaskCreator intentionally nil. Log: slog.New(slog.NewTextHandler(os.Stderr, nil)), } @@ -248,22 +249,31 @@ func TestDispatchBindingsNilDispatcherIsNoOp(t *testing.T) { // production uses (CREATE TABLE + ALTER TABLE migrations), so // binding_id / binding_version / binding_dispatches all exist on the // store by the time a test runs. -func openDispatchTestStore(t *testing.T) *store.Store { +// dispatchStores pairs the two stores the dispatcher spans: tasks stay on +// SQLite, event capture is the PocketBase store. They are embedded so a seed +// helper still reads as one handle; the method sets no longer overlap, which +// is exactly what the split bought. +type dispatchStores struct { + *store.Store + EdaStore *edastore.Store +} + +func openDispatchTestStore(t *testing.T) *dispatchStores { t.Helper() s, err := store.Open(t.Context(), filepath.Join(t.TempDir(), "dispatch.db")) if err != nil { t.Fatalf("store.Open: %v", err) } t.Cleanup(func() { _ = s.Close() }) - return s + return &dispatchStores{Store: s, EdaStore: edastore.OpenTest(t)} } // seedMapping inserts a mapping with the given fields and returns its id. // The Name field is "sentry-mapping" by default; callers pass fields // inline. -func seedMapping(t *testing.T, s *store.Store, sourceHint string, fields ...mapping.Field) int64 { +func seedMapping(t *testing.T, s *dispatchStores, sourceHint string, fields ...mapping.Field) string { t.Helper() - id, err := s.InsertMapping(t.Context(), mapping.Mapping{ + id, err := s.EdaStore.InsertMapping(t.Context(), mapping.Mapping{ Name: sourceHint + "-mapping", SourceHint: sourceHint, Fields: fields, @@ -281,7 +291,7 @@ func seedMapping(t *testing.T, s *store.Store, sourceHint string, fields ...mapp // advances through pending_approval via the store's test-only DB // accessor -- production callers must use the structured methods // (bindings.go) rather than touching the SQL directly. -func seedArmedBinding(t *testing.T, s *store.Store, source string, mappingID int64) (int64, int) { +func seedArmedBinding(t *testing.T, s *dispatchStores, source, mappingID string) (string, int) { t.Helper() return seedArmedBindingWithRepo(t, s, source, mappingID, "", "") } @@ -289,9 +299,9 @@ func seedArmedBinding(t *testing.T, s *store.Store, source string, mappingID int // seedArmedBindingWithRepo is seedArmedBinding with an explicit // owner/repo pin (pass "", "" for the unpinned case seedArmedBinding // covers). -func seedArmedBindingWithRepo(t *testing.T, s *store.Store, source string, mappingID int64, owner, repo string) (int64, int) { +func seedArmedBindingWithRepo(t *testing.T, s *dispatchStores, source, mappingID, owner, repo string) (string, int) { t.Helper() - id, err := s.InsertBinding(t.Context(), binding.Binding{ + id, err := s.EdaStore.InsertBinding(t.Context(), binding.Binding{ Name: "test " + source, Matcher: binding.Matcher{Source: source}, MappingID: mappingID, @@ -303,13 +313,25 @@ func seedArmedBindingWithRepo(t *testing.T, s *store.Store, source string, mappi if err != nil { t.Fatalf("InsertBinding: %v", err) } - if _, err := s.DB().ExecContext(t.Context(), `UPDATE bindings SET status='pending_approval' WHERE id=?`, id); err != nil { - t.Fatalf("force pending_approval: %v", err) + // Edit moves a draft to pending_approval through the real lifecycle, + // which is what makes it approvable. The old raw-SQL shortcut reached + // into a table the task store no longer owns. + if err := s.EdaStore.UpdateBinding(t.Context(), binding.Binding{ + ID: id, + Name: "test " + source, + Matcher: binding.Matcher{Source: source}, + MappingID: mappingID, + Workflow: "implement", + Owner: owner, + Repo: repo, + Secret: "0123456789abcdef0123456789abcdef", + }); err != nil { + t.Fatalf("UpdateBinding to pending_approval: %v", err) } - if err := s.ApproveBinding(t.Context(), id); err != nil { + if err := s.EdaStore.ApproveBinding(t.Context(), id); err != nil { t.Fatalf("ApproveBinding: %v", err) } - got, err := s.GetBinding(t.Context(), id) + got, err := s.EdaStore.GetBinding(t.Context(), id) if err != nil || got == nil { t.Fatalf("GetBinding after approve: %+v, %v", got, err) } @@ -321,9 +343,9 @@ func seedArmedBindingWithRepo(t *testing.T, s *store.Store, source string, mappi // the per-insert pruning the production path uses, so a test that // only needs a single row in the table is not at the mercy of // maxEvents=0's "no cap" reading. -func seedCapture(t *testing.T, s *store.Store, source string, authenticated bool, body string) { +func seedCapture(t *testing.T, s *dispatchStores, source string, authenticated bool, body string) { t.Helper() - _, err := s.InsertCapture(t.Context(), store.CapturedEvent{ + _, err := s.EdaStore.InsertCapture(t.Context(), store.CapturedEvent{ Source: source, Body: body, Authenticated: authenticated, @@ -341,7 +363,7 @@ func seedCapture(t *testing.T, s *store.Store, source string, authenticated bool // single-repo fallback accepts the dispatch; with zero or many // configured repos the loop would log and skip, masking the test's // positive assertions. -func newDispatchDaemon(t *testing.T, s *store.Store) *Daemon { +func newDispatchDaemon(t *testing.T, s *dispatchStores) *Daemon { t.Helper() return newDispatchDaemonWithRepos(t, s, []config.Repo{{Owner: "acme", Name: "widget"}}) } @@ -349,36 +371,37 @@ func newDispatchDaemon(t *testing.T, s *store.Store) *Daemon { // newDispatchDaemonWithRepos is newDispatchDaemon with an explicit repo // list, for tests exercising resolveBindingRepo's owner/repo-pin path // against zero, one, or several configured repos. -func newDispatchDaemonWithRepos(t *testing.T, s *store.Store, repos []config.Repo) *Daemon { +func newDispatchDaemonWithRepos(t *testing.T, s *dispatchStores, repos []config.Repo) *Daemon { t.Helper() cfg := config.Config{Repos: repos} return &Daemon{ Cfg: config.NewHolder(cfg), - Store: s, - Mappings: s, - Bindings: s, - BindingDispatcher: s, - BindingTaskCreator: s, + Store: s.Store, + Mappings: s.EdaStore, + Bindings: s.EdaStore, + BindingDispatcher: s.EdaStore, + BindingTaskCreator: s.Store, Log: slog.New(slog.NewTextHandler(os.Stderr, nil)), } } // assertDispatchRecorded reads the dedup ledger directly: the -// (binding_id, capture_id) primary key is what guarantees no double +// (binding, capture) unique index is what guarantees no double // dispatch. Going through the SQL table rather than the public // BindingStore surface keeps the assertion focused on the contract // Phase E actually writes to, not on a ListDispatches helper that // may not exist. -func assertDispatchRecorded(t *testing.T, s *store.Store, bindingID, taskID int64) { +func assertDispatchRecorded(t *testing.T, s *dispatchStores, bindingID string, taskID int64) { t.Helper() - var gotBindingID, gotTaskID int64 - if err := s.DB().QueryRowContext( - t.Context(), - `SELECT binding_id, task_id FROM binding_dispatches WHERE binding_id=?`, bindingID, - ).Scan(&gotBindingID, &gotTaskID); err != nil { + var gotBindingID string + var gotTaskID int64 + if err := s.EdaStore.App().DB().NewQuery( + "SELECT binding, task_id FROM binding_dispatches WHERE binding = {:binding}"). + Bind(map[string]any{"binding": bindingID}). + Row(&gotBindingID, &gotTaskID); err != nil { t.Fatalf("binding_dispatches row missing: %v", err) } if gotBindingID != bindingID || gotTaskID != taskID { - t.Fatalf("binding_dispatches row = (%d, %d), want (%d, %d)", gotBindingID, gotTaskID, bindingID, taskID) + t.Fatalf("binding_dispatches row = (%q, %d), want (%q, %d)", gotBindingID, gotTaskID, bindingID, taskID) } } diff --git a/internal/domain/binding/binding.go b/internal/domain/binding/binding.go index 92fa0a728..dc8f14112 100644 --- a/internal/domain/binding/binding.go +++ b/internal/domain/binding/binding.go @@ -43,10 +43,10 @@ type Matcher struct { // for an event to be marked authenticated; it is never returned by // GET handlers. type Binding struct { - ID int64 `json:"id"` + ID string `json:"id"` Name string `json:"name"` Matcher Matcher `json:"matcher"` - MappingID int64 `json:"mapping_id"` + MappingID string `json:"mapping_id"` Workflow string `json:"workflow"` // Owner and Repo pin a binding to a specific configured repo, so a // multi-repo deployment can dispatch correctly. Both empty means "no @@ -110,8 +110,8 @@ func (b Binding) validateCommon() error { if strings.TrimSpace(b.Matcher.Source) == "" { return fmt.Errorf("binding: matcher.source is required") } - if b.MappingID <= 0 { - return fmt.Errorf("binding: mapping_id must be positive") + if strings.TrimSpace(b.MappingID) == "" { + return fmt.Errorf("binding: mapping_id is required") } if strings.TrimSpace(b.Workflow) == "" { return fmt.Errorf("binding: workflow is required") diff --git a/internal/domain/binding/binding_test.go b/internal/domain/binding/binding_test.go index 7ca909086..793390ab9 100644 --- a/internal/domain/binding/binding_test.go +++ b/internal/domain/binding/binding_test.go @@ -6,7 +6,7 @@ func validBinding() Binding { return Binding{ Name: "sentry alerts", Matcher: Matcher{Source: "sentry"}, - MappingID: 1, + MappingID: "m1", Workflow: "implement", Secret: "0123456789abcdef0123456789abcdef", } diff --git a/internal/domain/mapping/mapping.go b/internal/domain/mapping/mapping.go index 3337fe6c9..c225210b0 100644 --- a/internal/domain/mapping/mapping.go +++ b/internal/domain/mapping/mapping.go @@ -50,7 +50,7 @@ type Field struct { // through one example captured event but not pinned to that event or its // source -- t2db.4's matcher decides which events a binding applies to. type Mapping struct { - ID int64 `json:"id"` + ID string `json:"id"` Name string `json:"name"` SourceHint string `json:"source_hint"` Fields []Field `json:"fields"` diff --git a/internal/domain/storecontract/storecontract.go b/internal/domain/storecontract/storecontract.go index 1956cee8a..2729d9638 100644 --- a/internal/domain/storecontract/storecontract.go +++ b/internal/domain/storecontract/storecontract.go @@ -94,7 +94,7 @@ type TaskEvents interface { // should not acquire the full task-lifecycle surface, mirroring why // TaskArchiver is split out above. See docs/prds/event-capture-storage.md. type CaptureStore interface { - InsertCapture(ctx context.Context, c CapturedEvent, retention time.Duration, maxEvents int) (int64, error) + InsertCapture(ctx context.Context, c CapturedEvent, retention time.Duration, maxEvents int) (string, error) ListCaptures(ctx context.Context, limit int) ([]CapturedEvent, error) } @@ -142,11 +142,11 @@ type ApplyStatusStore interface { // surface, not the full task or capture APIs. See // docs/prds/payload-field-mapping.md. type MappingStore interface { - InsertMapping(ctx context.Context, m mapping.Mapping) (int64, error) - GetMapping(ctx context.Context, id int64) (*mapping.Mapping, error) + InsertMapping(ctx context.Context, m mapping.Mapping) (string, error) + GetMapping(ctx context.Context, id string) (*mapping.Mapping, error) ListMappings(ctx context.Context) ([]mapping.Mapping, error) UpdateMapping(ctx context.Context, m mapping.Mapping) error - DeleteMapping(ctx context.Context, id int64) error + DeleteMapping(ctx context.Context, id string) error } // BindingStore persists playbook bindings (t2db.4 Phase B): CRUD and the @@ -156,12 +156,12 @@ type MappingStore interface { // BindingDispatcher; the daemon depends on both. See // docs/prds/playbook-binding.md. type BindingStore interface { - InsertBinding(ctx context.Context, b binding.Binding) (int64, error) - GetBinding(ctx context.Context, id int64) (*binding.Binding, error) + InsertBinding(ctx context.Context, b binding.Binding) (string, error) + GetBinding(ctx context.Context, id string) (*binding.Binding, error) ListBindings(ctx context.Context) ([]binding.Binding, error) UpdateBinding(ctx context.Context, b binding.Binding) error - DeleteBinding(ctx context.Context, id int64) error - ApproveBinding(ctx context.Context, id int64) error + DeleteBinding(ctx context.Context, id string) error + ApproveBinding(ctx context.Context, id string) error } // BindingDispatcher is the dispatch-loop surface over the bindings store: @@ -174,9 +174,11 @@ type BindingDispatcher interface { ArmedBindingsForSource(ctx context.Context, source string) ([]binding.Binding, error) RecordDispatch( ctx context.Context, - bindingID int64, + bindingID string, bindingVersion int64, - captureID int64, + captureID string, + // taskID addresses the SQLite-owned task tables, which are not + // migrating: it stays an integer on purpose. taskID int64, ) error ListUndispatchedCaptures(ctx context.Context, sources []string, limit int) ([]CapturedEvent, error) @@ -187,7 +189,7 @@ type BindingDispatcher interface { // keeps the lifecycle surface narrow (8 methods, the interfacebloat limit) // and keeps the binding-specific shape on the binding interfaces. type BindingTaskCreator interface { - EnqueueBindingTask(ctx context.Context, owner, repo, title, body, wf, identity string, bindingID int64, bindingVersion int) (*task.Task, error) + EnqueueBindingTask(ctx context.Context, owner, repo, title, body, wf, identity, bindingID string, bindingVersion int) (*task.Task, error) } // PlaybookDispatcher is the idempotency-ledger surface for side-effecting @@ -231,7 +233,7 @@ type TaskLogStore interface { // no forge/task association -- just what arrived, from where, and when. // See docs/prds/event-capture-storage.md. type CapturedEvent struct { - ID int64 `json:"id"` + ID string `json:"id"` ReceivedAt time.Time `json:"received_at"` Source string `json:"source"` RemoteAddr string `json:"remote_addr"` diff --git a/internal/domain/workflow/task/task.go b/internal/domain/workflow/task/task.go index 63e2e70a7..fe30ca1d8 100644 --- a/internal/domain/workflow/task/task.go +++ b/internal/domain/workflow/task/task.go @@ -82,8 +82,8 @@ type Task struct { // from a playbook binding dispatch (t2db.4 Phase B). They record // provenance: which binding fired this task and at what version, so // later edits to the binding cannot silently rewrite history. - BindingID int64 `json:"binding_id"` - BindingVersion int `json:"binding_version"` + BindingID string `json:"binding_id"` + BindingVersion int `json:"binding_version"` // ReviewPayload is the JSON-encoded review unit (the forge review's // actionable comments) the remediate workflow's current run must // address. The daemon's reaction consumer injects it before queuing a diff --git a/internal/infrastructure/captureintake/receiver_test.go b/internal/infrastructure/captureintake/receiver_test.go index 22c07def3..7889ff515 100644 --- a/internal/infrastructure/captureintake/receiver_test.go +++ b/internal/infrastructure/captureintake/receiver_test.go @@ -9,13 +9,13 @@ import ( "log/slog" "net/http" "net/http/httptest" - "path/filepath" "strings" "testing" "time" "github.com/samcharles93/archie-core/internal/domain/binding" "github.com/samcharles93/archie-core/internal/events" + "github.com/samcharles93/archie-core/internal/infrastructure/edastore" "github.com/samcharles93/archie-core/internal/store" "github.com/samcharles93/archie-core/internal/webhookguard" ) @@ -37,7 +37,9 @@ func (r *recordingPublisher) Publish(_ context.Context, e events.Event) { // wrote. type testReceiver struct { *Receiver - store *store.Store + // store is the event-capture store the receiver writes through, kept so a + // test asserts on persisted rows rather than on a mock's recollection. + store *edastore.Store } // Handler serves the receiver's route the way its host process mounts it, @@ -58,11 +60,7 @@ func captureTestServerWithoutStore(t *testing.T) testReceiver { func captureTestServer(t *testing.T) testReceiver { t.Helper() - s, err := store.Open(t.Context(), filepath.Join(t.TempDir(), "test.db")) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = s.Close() }) + s := edastore.OpenTest(t) return testReceiver{ Receiver: &Receiver{ Log: slog.New(slog.DiscardHandler), @@ -125,9 +123,9 @@ func (s *stubBindingStore) ApproveBinding(_ context.Context, _ int64) error { func (s *stubBindingStore) RecordDispatch( _ context.Context, + _ string, _ int64, - _ int64, - _ int64, + _ string, _ int64, ) error { return nil @@ -148,10 +146,10 @@ func captureBindingServer(t *testing.T, secret, source string) testReceiver { srv.Bindings = &stubBindingStore{ armedBySource: map[string][]binding.Binding{ source: {{ - ID: 1, + ID: "rbind0000000001", Name: "test", Matcher: binding.Matcher{Source: source}, - MappingID: 1, + MappingID: "rmap00000000001", Workflow: "implement", Version: 1, Status: binding.StatusArmed, @@ -425,8 +423,8 @@ type captureInsertErrorStore struct { store.CaptureStore } -func (captureInsertErrorStore) InsertCapture(context.Context, store.CapturedEvent, time.Duration, int) (int64, error) { - return 0, errors.New("boom") +func (captureInsertErrorStore) InsertCapture(context.Context, store.CapturedEvent, time.Duration, int) (string, error) { + return "", errors.New("boom") } func TestHandleCaptureStorageFailureIs500(t *testing.T) { @@ -543,8 +541,8 @@ func TestHandleCaptureWithMultipleArmedBindingsReturns409(t *testing.T) { srv.Bindings = &stubBindingStore{ armedBySource: map[string][]binding.Binding{ "github": { - {ID: 1, Matcher: binding.Matcher{Source: "github"}, Status: binding.StatusArmed, Secret: "abcdefghijklmnop"}, - {ID: 2, Matcher: binding.Matcher{Source: "github"}, Status: binding.StatusArmed, Secret: "qrstuvwxyz123456"}, + {ID: "rbind0000000001", Matcher: binding.Matcher{Source: "github"}, Status: binding.StatusArmed, Secret: "abcdefghijklmnop"}, + {ID: "rbind0000000002", Matcher: binding.Matcher{Source: "github"}, Status: binding.StatusArmed, Secret: "qrstuvwxyz123456"}, }, }, } diff --git a/internal/store/binding_cipher.go b/internal/infrastructure/edastore/cipher.go similarity index 99% rename from internal/store/binding_cipher.go rename to internal/infrastructure/edastore/cipher.go index ccbb27ec2..6c398adaf 100644 --- a/internal/store/binding_cipher.go +++ b/internal/infrastructure/edastore/cipher.go @@ -1,4 +1,4 @@ -package store +package edastore import ( "crypto/aes" diff --git a/internal/store/binding_cipher_test.go b/internal/infrastructure/edastore/cipher_test.go similarity index 58% rename from internal/store/binding_cipher_test.go rename to internal/infrastructure/edastore/cipher_test.go index 9ba199d2c..42ae13d26 100644 --- a/internal/store/binding_cipher_test.go +++ b/internal/infrastructure/edastore/cipher_test.go @@ -1,4 +1,4 @@ -package store +package edastore import ( "encoding/base64" @@ -121,122 +121,112 @@ func TestBindingCipherDerivesDistinctKeys(t *testing.T) { } } -func TestStoreBindingSecretEncryptedAtRest(t *testing.T) { - s := openTestWithBindingCipher(t) - b := testBinding("sentry") - id, err := s.InsertBinding(t.Context(), b) - if err != nil { - t.Fatalf("InsertBinding: %v", err) - } - - // The raw column must not be the plaintext secret. - var stored string - if err := s.db.QueryRowContext(t.Context(), `SELECT secret FROM bindings WHERE id=?`, id).Scan(&stored); err != nil { - t.Fatalf("read raw secret: %v", err) - } - if stored == b.Secret { - t.Fatal("bindings.secret stored as plaintext") - } - if !strings.HasPrefix(stored, "arcie-binding:v1:") { - t.Fatalf("stored secret = %q, want arcie-binding:v1: envelope", stored) - } - - // GetBinding returns the plaintext secret. - got, err := s.GetBinding(t.Context(), id) - if err != nil || got == nil { - t.Fatalf("GetBinding: %+v, %v", got, err) - } - if got.Secret != b.Secret { - t.Fatalf("GetBinding.Secret = %q, want %q", got.Secret, b.Secret) - } -} - -func TestStoreListBindingsDecryptsSecrets(t *testing.T) { - s := openTestWithBindingCipher(t) - if _, err := s.InsertBinding(t.Context(), testBinding("sentry")); err != nil { - t.Fatalf("InsertBinding: %v", err) - } - got, err := s.ListBindings(t.Context()) - if err != nil { - t.Fatalf("ListBindings: %v", err) - } - if len(got) != 1 { - t.Fatalf("ListBindings = %d rows, want 1", len(got)) - } - if got[0].Secret != testBinding("sentry").Secret { - t.Fatalf("ListBindings.Secret not decrypted") - } -} - -func TestStoreBindingCipherNoKeyIsPlaintext(t *testing.T) { - s := openTest(t) // no cipher -> legacy plaintext - b := testBinding("sentry") - id, err := s.InsertBinding(t.Context(), b) - if err != nil { - t.Fatalf("InsertBinding: %v", err) - } - got, err := s.GetBinding(t.Context(), id) - if err != nil || got == nil { - t.Fatalf("GetBinding: %+v, %v", got, err) - } - if got.Secret != b.Secret { - t.Fatalf("plaintext store round-trip = %q, want %q", got.Secret, b.Secret) - } -} - -// TestStoreEncryptedReadLeavesLegacyPlaintextAlone pins that enabling a -// cipher does not break rows written before it was configured: a non-envelope -// secret must remain readable as plaintext (encryption is not retroactive). -func TestStoreEncryptedReadLeavesLegacyPlaintextAlone(t *testing.T) { - path := filepath.Join(t.TempDir(), "legacy.db") - plain, err := Open(t.Context(), path) - if err != nil { - t.Fatalf("Open (plaintext): %v", err) - } - b := testBinding("sentry") - id, err := plain.InsertBinding(t.Context(), b) - if err != nil { - t.Fatalf("InsertBinding (plaintext): %v", err) - } - if err := plain.Close(); err != nil { - t.Fatalf("Close: %v", err) - } - - c, _ := NewBindingCipher(testBindingKey, nil) - encrypted, err := Open(t.Context(), path, WithBindingCipher(c)) - if err != nil { - t.Fatalf("Open (cipher): %v", err) - } - t.Cleanup(func() { _ = encrypted.Close() }) - got, err := encrypted.GetBinding(t.Context(), id) - if err != nil { - t.Fatalf("GetBinding under cipher: %v", err) - } - if got == nil { - t.Fatal("GetBinding returned nil") - } - if got.Secret != b.Secret { - t.Fatalf("legacy plaintext secret = %q, want %q (must stay readable)", got.Secret, b.Secret) - } -} - -func openTestWithBindingCipher(t *testing.T) *Store { +// openWithCipher builds a store whose binding secrets are encrypted at rest. +func openWithCipher(t *testing.T, cipher BindingCipher) *Store { t.Helper() - c, err := NewBindingCipher(testBindingKey, nil) + dir := t.TempDir() + s, err := Open(Config{ + DBPath: filepath.Join(dir, "eda.sqlite"), + DataDir: filepath.Join(dir, "pb_data"), + Cipher: cipher, + }) if err != nil { - t.Fatalf("NewBindingCipher: %v", err) - } - s, err := Open(t.Context(), filepath.Join(t.TempDir(), "test.db"), WithBindingCipher(c)) - if err != nil { - t.Fatalf("Open: %v", err) + t.Fatalf("Open() error = %v", err) } t.Cleanup(func() { _ = s.Close() }) return s } -// tamperEnvelope decodes the payload, flips one byte inside the sealed -// ciphertext (after the 12-byte nonce), and re-encodes it, so the auth tag -// must fail. It returns the envelope with a corrupt ciphertext. +func sampleBinding() bindingInput { + return bindingInput{Name: "n", Source: "sentry", Workflow: "implement", Secret: "supersecretvalue0123"} +} + +// TestBindingSecretIsEncryptedAtRest pins that the plaintext secret never +// reaches the database file. Reading the column directly is the point: a +// round-trip through the store would decrypt and prove nothing. +func TestBindingSecretIsEncryptedAtRest(t *testing.T) { + cipher, err := NewBindingCipher(testBindingKey, nil) + if err != nil { + t.Fatalf("NewBindingCipher() error = %v", err) + } + s := openWithCipher(t, cipher) + in := sampleBinding() + id := insertBinding(t, s, in) + + var stored string + if err := s.App().DB().NewQuery("SELECT secret FROM " + CollBindings + " WHERE id = {:id}"). + Bind(map[string]any{"id": id}).Row(&stored); err != nil { + t.Fatalf("read raw secret: %v", err) + } + if stored == in.Secret { + t.Fatal("secret is stored as plaintext; it must be encrypted at rest") + } + if !strings.Contains(stored, ":") { + t.Errorf("stored secret %q does not look like a cipher envelope", stored) + } +} + +// TestBindingSecretDecryptsOnRead pins the other half: what was encrypted +// comes back as the original plaintext. +func TestBindingSecretDecryptsOnRead(t *testing.T) { + cipher, _ := NewBindingCipher(testBindingKey, nil) + s := openWithCipher(t, cipher) + in := sampleBinding() + id := insertBinding(t, s, in) + + got, err := s.GetBinding(t.Context(), id) + if err != nil { + t.Fatalf("GetBinding() error = %v", err) + } + if got.Secret != in.Secret { + t.Errorf("Secret = %q, want %q", got.Secret, in.Secret) + } +} + +// TestNoCipherKeepsPlaintext pins the behaviour that predates encryption: a +// deployment with no configured key still works, storing plaintext. +func TestNoCipherKeepsPlaintext(t *testing.T) { + s := openWithCipher(t, nil) + in := sampleBinding() + id := insertBinding(t, s, in) + + var stored string + if err := s.App().DB().NewQuery("SELECT secret FROM " + CollBindings + " WHERE id = {:id}"). + Bind(map[string]any{"id": id}).Row(&stored); err != nil { + t.Fatalf("read raw secret: %v", err) + } + if stored != in.Secret { + t.Errorf("stored secret = %q, want the plaintext %q with no cipher configured", stored, in.Secret) + } +} + +// TestRotatedKeyStillReadsOldRows pins key rotation: a secret written under +// the previous key must stay readable once a new active key is installed, +// otherwise a rotation silently bricks every armed binding. +func TestRotatedKeyStillReadsOldRows(t *testing.T) { + const oldKey = "fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210" + oldCipher, _ := NewBindingCipher(oldKey, nil) + s := openWithCipher(t, oldCipher) + in := sampleBinding() + id := insertBinding(t, s, in) + + rotated, err := NewBindingCipher(testBindingKey, []string{oldKey}) + if err != nil { + t.Fatalf("NewBindingCipher(rotated) error = %v", err) + } + s.cipher = rotated + + got, err := s.GetBinding(t.Context(), id) + if err != nil { + t.Fatalf("GetBinding() after rotation error = %v", err) + } + if got.Secret != in.Secret { + t.Errorf("Secret after rotation = %q, want %q: the previous key must still decrypt", got.Secret, in.Secret) + } +} + +// tamperEnvelope flips one byte inside the ciphertext region so Decrypt must +// fail authentication rather than return altered plaintext. func tamperEnvelope(t *testing.T, envelope string) string { t.Helper() parts := strings.SplitN(envelope, ":", 4) diff --git a/internal/infrastructure/edastore/contracts.go b/internal/infrastructure/edastore/contracts.go new file mode 100644 index 000000000..ee1b1d9a9 --- /dev/null +++ b/internal/infrastructure/edastore/contracts.go @@ -0,0 +1,190 @@ +package edastore + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" + "time" + + "github.com/pocketbase/dbx" + "github.com/pocketbase/pocketbase/core" + + "github.com/samcharles93/archie-core/internal/domain/storecontract" +) + +// capturedAtLayout is fixed-width UTC with nanosecond precision, so string +// comparison is time comparison. time.RFC3339Nano trims trailing zeros, which +// would break that. +const capturedAtLayout = "2006-01-02T15:04:05.000000000Z" + +// This store is the implementation of every event-capture contract. The task +// lifecycle contracts stay on the SQLite store; the two meet only at +// binding_dispatches.task_id. +var ( + _ storecontract.CaptureStore = (*Store)(nil) + _ storecontract.MappingStore = (*Store)(nil) + _ storecontract.BindingStore = (*Store)(nil) + _ storecontract.BindingDispatcher = (*Store)(nil) + _ storecontract.PlaybookDispatcher = (*Store)(nil) +) + +func (s *Store) record(collection, id string) (*core.Record, error) { + r, err := s.app.FindRecordById(collection, id) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + return r, err +} + +// InsertCapture stores one inbound event verbatim. Redaction happens upstream +// in webhookguard; this persists exactly what it is given, because field +// mapping is designed against the payload as received. +// +// retention and maxEvents are accepted for contract compatibility and applied +// as a prune after the insert. +func (s *Store) InsertCapture(ctx context.Context, c storecontract.CapturedEvent, retention time.Duration, maxEvents int) (string, error) { + collection, err := s.app.FindCollectionByNameOrId(CollCaptures) + if err != nil { + return "", err + } + r := core.NewRecord(collection) + r.Set("source", c.Source) + r.Set("remote_addr", c.RemoteAddr) + r.Set("content_type", c.ContentType) + r.Set("headers", c.Headers) + r.Set("body", c.Body) + r.Set("authenticated", c.Authenticated) + received := c.ReceivedAt + if received.IsZero() { + received = time.Now() + } + r.Set("received_at", received.UTC().Format(capturedAtLayout)) + if err := s.app.Save(r); err != nil { + return "", fmt.Errorf("edastore: insert capture: %w", err) + } + if err := s.pruneCaptures(ctx, retention, maxEvents); err != nil { + return "", err + } + return r.Id, nil +} + +// pruneCaptures enforces the retention window and count cap. A capture exists +// to be mapped against while it is recent; keeping every event forever would +// make an unauthenticated intake surface a disk-growth vector. +func (s *Store) pruneCaptures(_ context.Context, retention time.Duration, maxEvents int) error { + if retention > 0 { + cutoff := time.Now().Add(-retention).UTC().Format(time.RFC3339) + if _, err := s.app.DB().NewQuery( + "DELETE FROM " + CollCaptures + " WHERE received_at < {:cutoff}"). + Bind(map[string]any{"cutoff": cutoff}).Execute(); err != nil { + return fmt.Errorf("edastore: prune captures by age: %w", err) + } + } + if maxEvents > 0 { + if _, err := s.app.DB().NewQuery( + "DELETE FROM " + CollCaptures + " WHERE id NOT IN (SELECT id FROM " + CollCaptures + + " ORDER BY received_at DESC LIMIT {:keep})"). + Bind(map[string]any{"keep": maxEvents}).Execute(); err != nil { + return fmt.Errorf("edastore: prune captures by count: %w", err) + } + } + return nil +} + +// parseCapturedAt reads the stored timestamp, tolerating the RFC3339 form a +// caller-supplied value may carry. +func parseCapturedAt(raw string) time.Time { + for _, layout := range []string{capturedAtLayout, time.RFC3339Nano, time.RFC3339} { + if t, err := time.Parse(layout, raw); err == nil { + return t + } + } + return time.Time{} +} + +func captureValue(r *core.Record) storecontract.CapturedEvent { + return storecontract.CapturedEvent{ + ID: r.Id, + ReceivedAt: parseCapturedAt(r.GetString("received_at")), + Source: r.GetString("source"), + RemoteAddr: r.GetString("remote_addr"), + ContentType: r.GetString("content_type"), + Headers: r.GetString("headers"), + Body: r.GetString("body"), + Authenticated: r.GetBool("authenticated"), + } +} + +// ListCaptures returns the newest captures first. +func (s *Store) ListCaptures(_ context.Context, limit int) ([]storecontract.CapturedEvent, error) { + records, err := s.app.FindRecordsByFilter(CollCaptures, "", "-received_at", limit, 0) + if err != nil { + return nil, fmt.Errorf("edastore: list captures: %w", err) + } + out := make([]storecontract.CapturedEvent, 0, len(records)) + for _, r := range records { + out = append(out, captureValue(r)) + } + return out, nil +} + +// ListUndispatchedCaptures returns recent captures for the given sources that +// no binding has dispatched yet. +// +// This is raw SQL rather than a record filter: the anti-join against the +// dispatch ledger is a subquery, and PocketBase's filter DSL is not SQL (it +// has no IN (SELECT ...) form). A read fires no hooks either way, so nothing +// is lost by dropping to the query builder here. +func (s *Store) ListUndispatchedCaptures(_ context.Context, sources []string, limit int) ([]storecontract.CapturedEvent, error) { + if len(sources) == 0 || limit <= 0 { + return nil, nil + } + placeholders := make([]string, 0, len(sources)) + params := dbx.Params{"limit": limit} + for i, src := range sources { + key := fmt.Sprintf("s%d", i) + placeholders = append(placeholders, "{:"+key+"}") + params[key] = src + } + query := "SELECT id, received_at, source, remote_addr, content_type, headers, body, authenticated FROM " + + CollCaptures + " WHERE source IN (" + strings.Join(placeholders, ",") + + ") AND id NOT IN (SELECT capture FROM " + CollBindingDispatches + ") ORDER BY received_at DESC LIMIT {:limit}" + + var rows []captureRow + if err := s.app.DB().NewQuery(query).Bind(params).All(&rows); err != nil { + return nil, fmt.Errorf("edastore: list undispatched captures: %w", err) + } + out := make([]storecontract.CapturedEvent, 0, len(rows)) + for _, r := range rows { + out = append(out, r.value()) + } + return out, nil +} + +// captureRow is the raw-SQL shape of a capture, used only where a record +// filter cannot express the query. +type captureRow struct { + ID string `db:"id"` + ReceivedAt string `db:"received_at"` + Source string `db:"source"` + RemoteAddr string `db:"remote_addr"` + ContentType string `db:"content_type"` + Headers string `db:"headers"` + Body string `db:"body"` + Authenticated bool `db:"authenticated"` +} + +func (r captureRow) value() storecontract.CapturedEvent { + return storecontract.CapturedEvent{ + ID: r.ID, + ReceivedAt: parseCapturedAt(r.ReceivedAt), + Source: r.Source, + RemoteAddr: r.RemoteAddr, + ContentType: r.ContentType, + Headers: r.Headers, + Body: r.Body, + Authenticated: r.Authenticated, + } +} diff --git a/internal/infrastructure/edastore/contracts_binding.go b/internal/infrastructure/edastore/contracts_binding.go new file mode 100644 index 000000000..cc61cb4ad --- /dev/null +++ b/internal/infrastructure/edastore/contracts_binding.go @@ -0,0 +1,370 @@ +package edastore + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/pocketbase/pocketbase/core" + + "github.com/samcharles93/archie-core/internal/domain/binding" + "github.com/samcharles93/archie-core/internal/domain/mapping" +) + +// --- mappings --- + +func mappingValue(r *core.Record) (mapping.Mapping, error) { + var fields []mapping.Field + if raw := r.GetString("fields"); raw != "" { + if err := json.Unmarshal([]byte(raw), &fields); err != nil { + return mapping.Mapping{}, fmt.Errorf("edastore: decode mapping fields: %w", err) + } + } + return mapping.Mapping{ + ID: r.Id, + Name: r.GetString("name"), + SourceHint: r.GetString("source_hint"), + Fields: fields, + CreatedAt: r.GetDateTime("created_at").Time(), + UpdatedAt: r.GetDateTime("updated_at").Time(), + }, nil +} + +func (s *Store) applyMapping(r *core.Record, m mapping.Mapping) error { + encoded, err := json.Marshal(m.Fields) + if err != nil { + return fmt.Errorf("edastore: encode mapping fields: %w", err) + } + r.Set("name", m.Name) + r.Set("source_hint", m.SourceHint) + r.Set("fields", string(encoded)) + return nil +} + +func (s *Store) InsertMapping(_ context.Context, m mapping.Mapping) (string, error) { + collection, err := s.app.FindCollectionByNameOrId(CollMappings) + if err != nil { + return "", err + } + r := core.NewRecord(collection) + if err := s.applyMapping(r, m); err != nil { + return "", err + } + if err := s.app.Save(r); err != nil { + return "", fmt.Errorf("edastore: insert mapping: %w", err) + } + return r.Id, nil +} + +func (s *Store) GetMapping(_ context.Context, id string) (*mapping.Mapping, error) { + r, err := s.record(CollMappings, id) + if err != nil || r == nil { + return nil, err + } + m, err := mappingValue(r) + if err != nil { + return nil, err + } + return &m, nil +} + +func (s *Store) ListMappings(_ context.Context) ([]mapping.Mapping, error) { + records, err := s.app.FindRecordsByFilter(CollMappings, "", "-created_at", 0, 0) + if err != nil { + return nil, fmt.Errorf("edastore: list mappings: %w", err) + } + out := make([]mapping.Mapping, 0, len(records)) + for _, r := range records { + m, err := mappingValue(r) + if err != nil { + return nil, err + } + out = append(out, m) + } + return out, nil +} + +func (s *Store) UpdateMapping(_ context.Context, m mapping.Mapping) error { + r, err := s.record(CollMappings, m.ID) + if err != nil { + return err + } + if r == nil { + return ErrMappingNotFound + } + if err := s.applyMapping(r, m); err != nil { + return err + } + return s.app.Save(r) +} + +func (s *Store) DeleteMapping(_ context.Context, id string) error { + r, err := s.record(CollMappings, id) + if err != nil { + return err + } + if r == nil { + return ErrMappingNotFound + } + return s.app.Delete(r) +} + +// --- bindings --- + +func (s *Store) bindingValue(r *core.Record) (binding.Binding, error) { + b := binding.Binding{ + ID: r.Id, + Name: r.GetString("name"), + Matcher: binding.Matcher{Source: r.GetString("source")}, + MappingID: r.GetString("mapping"), + Workflow: r.GetString("workflow"), + Owner: r.GetString("owner"), + Repo: r.GetString("repo"), + Version: r.GetInt("version"), + Status: binding.Status(r.GetString("status")), + Secret: r.GetString("secret"), + CreatedAt: r.GetDateTime("created_at").Time(), + UpdatedAt: r.GetDateTime("updated_at").Time(), + } + if s.cipher != nil && b.Secret != "" { + plain, err := s.cipher.Decrypt(b.Secret) + if err != nil { + return binding.Binding{}, fmt.Errorf("edastore: decrypt binding secret: %w", err) + } + b.Secret = plain + } + return b, nil +} + +func (s *Store) applyBinding(r *core.Record, b binding.Binding) error { + r.Set("name", b.Name) + r.Set("source", b.Matcher.Source) + r.Set("mapping", b.MappingID) + r.Set("workflow", b.Workflow) + r.Set("owner", b.Owner) + r.Set("repo", b.Repo) + // An empty secret means "keep the stored one". An edit form that does not + // echo the secret back must not silently disarm the binding by erasing it. + if b.Secret == "" { + return nil + } + secret := b.Secret + if s.cipher != nil { + encrypted, err := s.cipher.Encrypt(secret) + if err != nil { + return fmt.Errorf("edastore: encrypt binding secret: %w", err) + } + secret = encrypted + } + r.Set("secret", secret) + return nil +} + +// InsertBinding stores a new binding as a draft. The caller's status is +// ignored on purpose: a binding cannot go live without explicit approval, so +// ApproveBinding is the only path to armed. +// +// A source may carry at most one binding, so an insert against a source that +// already has one is refused rather than creating an ambiguous second matcher. +func (s *Store) InsertBinding(ctx context.Context, b binding.Binding) (string, error) { + existing, err := s.bindingsByFilter("source = {:source}", map[string]any{"source": b.Matcher.Source}) + if err != nil { + return "", err + } + if len(existing) > 0 { + return "", ErrBindingOverlap + } + collection, err := s.app.FindCollectionByNameOrId(CollBindings) + if err != nil { + return "", err + } + r := core.NewRecord(collection) + if err := s.applyBinding(r, b); err != nil { + return "", err + } + r.Set("version", 1) + r.Set("status", string(binding.StatusDraft)) + if err := s.app.Save(r); err != nil { + return "", fmt.Errorf("edastore: insert binding: %w", err) + } + _ = ctx + return r.Id, nil +} + +func (s *Store) GetBinding(_ context.Context, id string) (*binding.Binding, error) { + r, err := s.record(CollBindings, id) + if err != nil || r == nil { + return nil, err + } + b, err := s.bindingValue(r) + if err != nil { + return nil, err + } + return &b, nil +} + +func (s *Store) ListBindings(_ context.Context) ([]binding.Binding, error) { + return s.bindingsByFilter("", nil) +} + +// ArmedBindingsForSource returns the armed bindings a source can dispatch. A +// draft or pending_approval binding is never returned: that is the gate. +func (s *Store) ArmedBindingsForSource(_ context.Context, source string) ([]binding.Binding, error) { + return s.bindingsByFilter("source = {:source} && status = {:status}", + map[string]any{"source": source, "status": string(binding.StatusArmed)}) +} + +func (s *Store) bindingsByFilter(filter string, params map[string]any) ([]binding.Binding, error) { + var records []*core.Record + var err error + if params == nil { + records, err = s.app.FindRecordsByFilter(CollBindings, filter, "-created_at", 0, 0) + } else { + records, err = s.app.FindRecordsByFilter(CollBindings, filter, "-created_at", 0, 0, params) + } + if err != nil { + return nil, fmt.Errorf("edastore: list bindings: %w", err) + } + out := make([]binding.Binding, 0, len(records)) + for _, r := range records { + b, err := s.bindingValue(r) + if err != nil { + return nil, err + } + out = append(out, b) + } + return out, nil +} + +// UpdateBinding rewrites a binding's editable fields, bumps its version, and +// sends it back to pending_approval. An edit always requires re-approval: an +// armed binding whose matcher or workflow changed is a different binding, and +// re-arming it silently would defeat the gate. +func (s *Store) UpdateBinding(_ context.Context, b binding.Binding) error { + r, err := s.record(CollBindings, b.ID) + if err != nil { + return err + } + if r == nil { + return ErrBindingNotFound + } + others, err := s.bindingsByFilter("source = {:source} && id != {:id}", + map[string]any{"source": b.Matcher.Source, "id": b.ID}) + if err != nil { + return err + } + if len(others) > 0 { + return ErrBindingOverlap + } + if err := s.applyBinding(r, b); err != nil { + return err + } + r.Set("status", string(binding.StatusPendingApproval)) + r.Set("version", r.GetInt("version")+1) + return s.app.Save(r) +} + +func (s *Store) DeleteBinding(_ context.Context, id string) error { + r, err := s.record(CollBindings, id) + if err != nil { + return err + } + if r == nil { + return ErrBindingNotFound + } + return s.app.Delete(r) +} + +// ApproveBinding is the only transition that arms a binding. It moves +// pending_approval to armed; any other current state is refused, so a draft +// cannot skip review and an armed binding cannot be re-approved. +// +// A source may have only one armed binding, so approval is refused when +// another binding on the same source is already armed. Without that guard, +// one capture could match two armed matchers and dispatch twice. +func (s *Store) ApproveBinding(_ context.Context, id string) error { + r, err := s.record(CollBindings, id) + if err != nil { + return err + } + if r == nil { + return ErrBindingNotFound + } + armed, err := s.bindingsByFilter("source = {:source} && status = {:status} && id != {:id}", + map[string]any{"source": r.GetString("source"), "status": string(binding.StatusArmed), "id": id}) + if err != nil { + return err + } + if len(armed) > 0 { + return ErrBindingOverlap + } + if r.GetString("status") != string(binding.StatusPendingApproval) { + return ErrBindingTransition + } + r.Set("status", string(binding.StatusArmed)) + return s.app.Save(r) +} + +// --- dispatch ledgers --- + +// RecordDispatch writes one at-most-once binding dispatch. Uniqueness is the +// collection's index, so a replay is refused by the schema; the caller sees +// ErrAlreadyDispatched rather than a silent second run. +func (s *Store) RecordDispatch(_ context.Context, bindingID string, bindingVersion int64, captureID string, taskID int64) error { + collection, err := s.app.FindCollectionByNameOrId(CollBindingDispatches) + if err != nil { + return err + } + r := core.NewRecord(collection) + r.Set("binding", bindingID) + r.Set("binding_version", bindingVersion) + r.Set("capture", captureID) + r.Set("task_id", taskID) + return ledgerWrite(s.app.Save(r), "edastore: record dispatch") +} + +func (s *Store) RecordPlaybookDispatch(_ context.Context, playbookID, playbookVersion, eventID, actionID string) error { + collection, err := s.app.FindCollectionByNameOrId(CollPlaybookDispatches) + if err != nil { + return err + } + r := core.NewRecord(collection) + r.Set("playbook_id", playbookID) + r.Set("playbook_version", playbookVersion) + r.Set("event_id", eventID) + r.Set("action_id", actionID) + return ledgerWrite(s.app.Save(r), "edastore: record playbook dispatch") +} + +func (s *Store) DeletePlaybookDispatches(_ context.Context, playbookID string) error { + _, err := s.app.DB().NewQuery( + "DELETE FROM " + CollPlaybookDispatches + " WHERE playbook_id = {:id}"). + Bind(map[string]any{"id": playbookID}).Execute() + if err != nil { + return fmt.Errorf("edastore: delete playbook dispatches: %w", err) + } + return nil +} + +// ledgerWrite turns the unique-index refusal into ErrAlreadyDispatched and +// leaves every other failure alone. Treating all errors as "already recorded" +// would turn an at-most-once ledger into a silent at-least-once dispatcher. +func ledgerWrite(err error, what string) error { + switch { + case err == nil: + return nil + case isUniqueViolation(err): + return ErrAlreadyDispatched + default: + return fmt.Errorf("%s: %w", what, err) + } +} + +// isUniqueViolation reports whether err is the unique-index refusal. +// PocketBase surfaces it as a validation failure rather than a typed error, +// so the message is what there is to match on. +func isUniqueViolation(err error) bool { + text := strings.ToLower(err.Error()) + return strings.Contains(text, "unique") || strings.Contains(text, "already exists") +} diff --git a/internal/infrastructure/edastore/edastore.go b/internal/infrastructure/edastore/edastore.go index 2adca1db6..15d2922ff 100644 --- a/internal/infrastructure/edastore/edastore.go +++ b/internal/infrastructure/edastore/edastore.go @@ -27,6 +27,8 @@ import ( "github.com/pocketbase/pocketbase" "github.com/pocketbase/pocketbase/core" _ "modernc.org/sqlite" + + "github.com/samcharles93/archie-core/internal/domain/storecontract" ) // Collection names. They are unprefixed because this store owns its database; @@ -40,19 +42,21 @@ const ( CollToolCalls = "tool_calls" ) -// Binding lifecycle. A binding is only live once a human approves it, which is -// an acceptance criterion of the epic rather than a convention. -const ( - StatusDraft = "draft" - StatusApproved = "approved" +// Sentinels are re-exported from the contract rather than redefined. Their +// exact message strings are the wire contract: staterpc matches on (code, +// canonical message) to rehydrate them on the client, so a second definition +// with different text would break errors.Is across the gRPC boundary without +// changing anything visible. +var ( + ErrBindingNotFound = storecontract.ErrBindingNotFound + ErrBindingOverlap = storecontract.ErrBindingOverlap + ErrBindingTransition = storecontract.ErrBindingTransition + ErrAlreadyDispatched = storecontract.ErrAlreadyDispatched + ErrMappingNotFound = storecontract.ErrMappingNotFound ) -var ( - // ErrBindingNotFound reports an operation naming a binding that is absent. - ErrBindingNotFound = errors.New("edastore: binding not found") - // ErrCaptureNotFound reports a capture that is absent or aged out. - ErrCaptureNotFound = errors.New("edastore: capture not found") -) +// ErrCaptureNotFound reports a capture that is absent or aged out. +var ErrCaptureNotFound = errors.New("edastore: capture not found") // Collections returns every collection this store owns. func Collections() []string { @@ -64,6 +68,9 @@ func Collections() []string { // Config describes one store. type Config struct { + // Cipher encrypts binding secrets at rest. Nil keeps them plaintext, + // which is the behaviour that predates the option. + Cipher BindingCipher // DBPath is the SQLite file PocketBase owns for these collections. DBPath string // DataDir holds PocketBase's auxiliary database and settings. @@ -71,7 +78,10 @@ type Config struct { } // Store is the PocketBase-backed EDA persistence. -type Store struct{ app *pocketbase.PocketBase } +type Store struct { + app *pocketbase.PocketBase + cipher BindingCipher +} // Open bootstraps the store and ensures its collections exist. func Open(cfg Config) (*Store, error) { @@ -100,7 +110,7 @@ func Open(cfg Config) (*Store, error) { if err := ensureCollections(app); err != nil { return nil, err } - return &Store{app: app}, nil + return &Store{app: app, cipher: cfg.Cipher}, nil } // App exposes the PocketBase application so one process can serve this store diff --git a/internal/infrastructure/edastore/opentest.go b/internal/infrastructure/edastore/opentest.go new file mode 100644 index 000000000..2168e553e --- /dev/null +++ b/internal/infrastructure/edastore/opentest.go @@ -0,0 +1,23 @@ +package edastore + +import ( + "path/filepath" + "testing" +) + +// OpenTest builds a throwaway store for tests, closed when the test ends. +// It mirrors store.OpenTest so a test that needs both stores wires them the +// same way. +func OpenTest(t *testing.T) *Store { + t.Helper() + dir := t.TempDir() + s, err := Open(Config{ + DBPath: filepath.Join(dir, "eda.sqlite"), + DataDir: filepath.Join(dir, "pb_data"), + }) + if err != nil { + t.Fatalf("edastore.OpenTest: %v", err) + } + t.Cleanup(func() { _ = s.Close() }) + return s +} diff --git a/internal/infrastructure/edastore/records.go b/internal/infrastructure/edastore/records.go deleted file mode 100644 index f079c7830..000000000 --- a/internal/infrastructure/edastore/records.go +++ /dev/null @@ -1,211 +0,0 @@ -package edastore - -import ( - "context" - "database/sql" - "errors" - "fmt" - "strings" - - "github.com/pocketbase/pocketbase/core" -) - -// Capture is one inbound event, stored whether or not anything understands it. -type Capture struct { - ID string - Source string - RemoteAddr string - ContentType string - Headers string - Body string - Authenticated bool -} - -// Binding ties a matcher and mapping to a workflow. -type Binding struct { - ID string - Name string - Source string - MappingID string - Workflow string - Owner string - Repo string - Version int64 - Status string - Secret string -} - -// Dispatch is one at-most-once binding dispatch ledger entry. -type Dispatch struct { - BindingID string - BindingVersion int64 - CaptureID string - // TaskID addresses the SQLite-owned task tables, which are not migrating. - TaskID int64 -} - -// PlaybookDispatch is one at-most-once playbook action ledger entry. -type PlaybookDispatch struct { - PlaybookID string - PlaybookVersion string - EventID string - ActionID string -} - -// InsertCapture stores one inbound event and returns its id. -func (s *Store) InsertCapture(_ context.Context, c Capture) (string, error) { - collection, err := s.app.FindCollectionByNameOrId(CollCaptures) - if err != nil { - return "", err - } - r := core.NewRecord(collection) - r.Set("source", c.Source) - r.Set("remote_addr", c.RemoteAddr) - r.Set("content_type", c.ContentType) - r.Set("headers", c.Headers) - r.Set("body", c.Body) - r.Set("authenticated", c.Authenticated) - if err := s.app.Save(r); err != nil { - return "", fmt.Errorf("edastore: insert capture: %w", err) - } - return r.Id, nil -} - -// Capture returns one stored event verbatim. -func (s *Store) Capture(_ context.Context, id string) (Capture, error) { - r, err := s.app.FindRecordById(CollCaptures, id) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return Capture{}, ErrCaptureNotFound - } - return Capture{}, err - } - return Capture{ - ID: r.Id, - Source: r.GetString("source"), - RemoteAddr: r.GetString("remote_addr"), - ContentType: r.GetString("content_type"), - Headers: r.GetString("headers"), - Body: r.GetString("body"), - Authenticated: r.GetBool("authenticated"), - }, nil -} - -// InsertBinding stores a new binding. It is always created as a draft: the -// epic requires explicit human approval before a binding can run anything. -func (s *Store) InsertBinding(_ context.Context, b Binding) (string, error) { - collection, err := s.app.FindCollectionByNameOrId(CollBindings) - if err != nil { - return "", err - } - r := core.NewRecord(collection) - r.Set("name", b.Name) - r.Set("source", b.Source) - r.Set("mapping", b.MappingID) - r.Set("workflow", b.Workflow) - r.Set("owner", b.Owner) - r.Set("repo", b.Repo) - r.Set("version", 1) - r.Set("status", StatusDraft) - r.Set("secret", b.Secret) - if err := s.app.Save(r); err != nil { - return "", fmt.Errorf("edastore: insert binding: %w", err) - } - return r.Id, nil -} - -// GetBinding returns one binding. -func (s *Store) GetBinding(_ context.Context, id string) (Binding, error) { - r, err := s.app.FindRecordById(CollBindings, id) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return Binding{}, ErrBindingNotFound - } - return Binding{}, err - } - return Binding{ - ID: r.Id, - Name: r.GetString("name"), - Source: r.GetString("source"), - MappingID: r.GetString("mapping"), - Workflow: r.GetString("workflow"), - Owner: r.GetString("owner"), - Repo: r.GetString("repo"), - Version: int64(r.GetInt("version")), - Status: r.GetString("status"), - Secret: r.GetString("secret"), - }, nil -} - -// ApproveBinding moves a draft binding to approved. It is the only way a -// binding becomes live. -func (s *Store) ApproveBinding(_ context.Context, id string) error { - r, err := s.app.FindRecordById(CollBindings, id) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return ErrBindingNotFound - } - return err - } - r.Set("status", StatusApproved) - r.Set("version", r.GetInt("version")+1) - if err := s.app.Save(r); err != nil { - return fmt.Errorf("edastore: approve binding: %w", err) - } - return nil -} - -// RecordDispatch writes one binding dispatch, reporting whether it was new. -// The (binding, capture) uniqueness is enforced by the collection's index, so -// a duplicate is refused by the schema rather than by the caller. -func (s *Store) RecordDispatch(_ context.Context, d Dispatch) (bool, error) { - collection, err := s.app.FindCollectionByNameOrId(CollBindingDispatches) - if err != nil { - return false, err - } - r := core.NewRecord(collection) - r.Set("binding", d.BindingID) - r.Set("binding_version", d.BindingVersion) - r.Set("capture", d.CaptureID) - r.Set("task_id", d.TaskID) - return savedOrDuplicate(s.app, r, "edastore: record dispatch") -} - -// RecordPlaybookDispatch writes one playbook action dispatch, reporting -// whether it was new. -func (s *Store) RecordPlaybookDispatch(_ context.Context, d PlaybookDispatch) (bool, error) { - collection, err := s.app.FindCollectionByNameOrId(CollPlaybookDispatches) - if err != nil { - return false, err - } - r := core.NewRecord(collection) - r.Set("playbook_id", d.PlaybookID) - r.Set("playbook_version", d.PlaybookVersion) - r.Set("event_id", d.EventID) - r.Set("action_id", d.ActionID) - return savedOrDuplicate(s.app, r, "edastore: record playbook dispatch") -} - -// savedOrDuplicate saves a ledger record, translating the unique-index -// violation into "already recorded" rather than an error. Anything else is a -// real failure and is returned: silently swallowing every save error here -// would turn a broken ledger into a silent at-least-once dispatcher. -func savedOrDuplicate(app core.App, r *core.Record, what string) (bool, error) { - err := app.Save(r) - switch { - case err == nil: - return true, nil - case isUniqueViolation(err): - return false, nil - default: - return false, fmt.Errorf("%s: %w", what, err) - } -} - -// isUniqueViolation reports whether err is the unique-index refusal. PocketBase -// surfaces it as a validation failure rather than a typed error, so the text is -// what there is to match on. -func isUniqueViolation(err error) bool { - text := strings.ToLower(err.Error()) - return strings.Contains(text, "unique") || strings.Contains(text, "already exists") -} diff --git a/internal/infrastructure/edastore/schema.go b/internal/infrastructure/edastore/schema.go index f7c83f0ad..9d125b24c 100644 --- a/internal/infrastructure/edastore/schema.go +++ b/internal/infrastructure/edastore/schema.go @@ -7,6 +7,8 @@ import ( "path/filepath" "github.com/pocketbase/pocketbase/core" + + "github.com/samcharles93/archie-core/internal/domain/binding" ) // isDataDB reports whether PocketBase is asking for its main database, as @@ -73,7 +75,13 @@ func ensureCaptures(app core.App) error { &core.JSONField{Name: "headers", MaxSize: 1 << 20}, &core.TextField{Name: "body", Max: 1 << 20}, &core.BoolField{Name: "authenticated"}, - &core.AutodateField{Name: "received_at", OnCreate: true}, + // received_at is written by the store, not an autodate field, because + // it is also the sort key. PocketBase autodates are millisecond + // precision and record ids are random, so two captures arriving in + // the same millisecond would order arbitrarily -- the old store got + // insertion order free from an autoincrement id. capturedAtLayout is + // fixed-width so lexicographic order is chronological order. + &core.TextField{Name: "received_at"}, ) c.AddIndex("idx_captures_source", false, "source, received_at", "") return save(app, c) @@ -111,9 +119,14 @@ func ensureBindings(app core.App) error { &core.TextField{Name: "owner"}, &core.TextField{Name: "repo"}, &core.NumberField{Name: "version"}, - // status carries the approval gate. It is deliberately not defaulted - // to anything live: a binding reaches "approved" only by ApproveBinding. - &core.SelectField{Name: "status", Values: []string{StatusDraft, StatusApproved}, MaxSelect: 1}, + // status is the domain's three-state lifecycle, not a local + // vocabulary: draft -> pending_approval -> armed. Only + // ApproveBinding reaches armed, which is the epic's approval gate. + &core.SelectField{Name: "status", MaxSelect: 1, Values: []string{ + string(binding.StatusDraft), + string(binding.StatusPendingApproval), + string(binding.StatusArmed), + }}, &core.TextField{Name: "secret"}, &core.AutodateField{Name: "created_at", OnCreate: true}, &core.AutodateField{Name: "updated_at", OnCreate: true, OnUpdate: true}, diff --git a/internal/infrastructure/edastore/schema_test.go b/internal/infrastructure/edastore/schema_test.go index 242d64534..798d2c4e0 100644 --- a/internal/infrastructure/edastore/schema_test.go +++ b/internal/infrastructure/edastore/schema_test.go @@ -7,6 +7,8 @@ import ( "github.com/pocketbase/pocketbase/core" + "github.com/samcharles93/archie-core/internal/domain/binding" + "github.com/samcharles93/archie-core/internal/domain/storecontract" "github.com/samcharles93/archie-core/internal/infrastructure/edastore" ) @@ -46,23 +48,13 @@ func TestCollectionsAreCreated(t *testing.T) { // refused by the schema rather than by the caller remembering to ignore it. func TestBindingDispatchIsIdempotent(t *testing.T) { st := newStore(t) - first, err := st.RecordDispatch(t.Context(), edastore.Dispatch{ - BindingID: "b1", BindingVersion: 1, CaptureID: "c1", TaskID: 42, - }) - if err != nil { + if err := st.RecordDispatch(t.Context(), "b1", 1, "c1", 42); err != nil { t.Fatalf("first RecordDispatch() error = %v", err) } - if !first { - t.Fatal("first RecordDispatch() = false, want true: a fresh dispatch must be recorded") - } - second, err := st.RecordDispatch(t.Context(), edastore.Dispatch{ - BindingID: "b1", BindingVersion: 1, CaptureID: "c1", TaskID: 99, - }) - if err != nil { - t.Fatalf("second RecordDispatch() error = %v", err) - } - if second { - t.Error("second RecordDispatch() = true, want false: (binding, capture) is an at-most-once ledger") + // A replay of the same (binding, capture) must be refused, not run again. + err := st.RecordDispatch(t.Context(), "b1", 1, "c1", 99) + if !errors.Is(err, storecontract.ErrAlreadyDispatched) { + t.Errorf("second RecordDispatch() error = %v, want ErrAlreadyDispatched: (binding, capture) is an at-most-once ledger", err) } } @@ -70,12 +62,12 @@ func TestBindingDispatchIsIdempotent(t *testing.T) { // four-part playbook key (playbook, version, event, action). func TestPlaybookDispatchIsIdempotent(t *testing.T) { st := newStore(t) - key := edastore.PlaybookDispatch{PlaybookID: "p1", PlaybookVersion: "v1", EventID: "e1", ActionID: "a1"} - if ok, err := st.RecordPlaybookDispatch(t.Context(), key); err != nil || !ok { - t.Fatalf("first RecordPlaybookDispatch() = %v, %v; want true, nil", ok, err) + if err := st.RecordPlaybookDispatch(t.Context(), "p1", "v1", "e1", "a1"); err != nil { + t.Fatalf("first RecordPlaybookDispatch() error = %v", err) } - if ok, err := st.RecordPlaybookDispatch(t.Context(), key); err != nil || ok { - t.Fatalf("second RecordPlaybookDispatch() = %v, %v; want false, nil", ok, err) + err := st.RecordPlaybookDispatch(t.Context(), "p1", "v1", "e1", "a1") + if !errors.Is(err, storecontract.ErrAlreadyDispatched) { + t.Errorf("second RecordPlaybookDispatch() error = %v, want ErrAlreadyDispatched", err) } } @@ -84,7 +76,9 @@ func TestPlaybookDispatchIsIdempotent(t *testing.T) { // binding created without a status must not default to anything live. func TestBindingStartsAsDraft(t *testing.T) { st := newStore(t) - id, err := st.InsertBinding(t.Context(), edastore.Binding{Name: "n", Source: "sentry", Workflow: "implement"}) + id, err := st.InsertBinding(t.Context(), binding.Binding{ + Name: "n", Matcher: binding.Matcher{Source: "sentry"}, Workflow: "implement", + }) if err != nil { t.Fatalf("InsertBinding() error = %v", err) } @@ -92,15 +86,15 @@ func TestBindingStartsAsDraft(t *testing.T) { if err != nil { t.Fatalf("GetBinding() error = %v", err) } - if b.Status != edastore.StatusDraft { - t.Errorf("new binding status = %q, want %q: a binding must not be live before approval", b.Status, edastore.StatusDraft) + if b.Status != binding.StatusDraft { + t.Errorf("new binding status = %q, want %q: a binding must not be live before approval", b.Status, binding.StatusDraft) } } // TestApproveRejectsUnknownBinding pins the sentinel the gRPC layer maps. func TestApproveRejectsUnknownBinding(t *testing.T) { st := newStore(t) - if err := st.ApproveBinding(t.Context(), "nosuchbinding"); !errors.Is(err, edastore.ErrBindingNotFound) { + if err := st.ApproveBinding(t.Context(), "nosuchbinding"); !errors.Is(err, storecontract.ErrBindingNotFound) { t.Errorf("ApproveBinding(unknown) error = %v, want ErrBindingNotFound", err) } } @@ -111,19 +105,90 @@ func TestApproveRejectsUnknownBinding(t *testing.T) { func TestCaptureRoundTripsPayload(t *testing.T) { st := newStore(t) body := `{"action":"created","issue":{"number":7,"title":"it broke"}}` - id, err := st.InsertCapture(t.Context(), edastore.Capture{ + if _, err := st.InsertCapture(t.Context(), storecontract.CapturedEvent{ Source: "sentry", Body: body, Headers: `{"X-Hook":"1"}`, ContentType: "application/json", - }) - if err != nil { + }, 0, 0); err != nil { t.Fatalf("InsertCapture() error = %v", err) } - got, err := st.Capture(t.Context(), id) + list, err := st.ListCaptures(t.Context(), 10) if err != nil { - t.Fatalf("Capture() error = %v", err) + t.Fatalf("ListCaptures() error = %v", err) } + if len(list) != 1 { + t.Fatalf("ListCaptures() returned %d captures, want 1", len(list)) + } + got := list[0] if got.Body != body { t.Errorf("Body round-trip = %q, want %q: the raw payload is what field mapping is designed against", got.Body, body) } } var _ = core.Collection{} + +// TestBindingLifecycleIsThreeStates pins draft -> pending_approval -> armed. +// The states are the domain's, not this package's: collapsing them (as an +// earlier version of this store did) removes the review step while still +// looking like an approval gate, which is the worst of both. +func TestBindingLifecycleIsThreeStates(t *testing.T) { + st := newStore(t) + id, err := st.InsertBinding(t.Context(), binding.Binding{ + Name: "n", Matcher: binding.Matcher{Source: "sentry"}, Workflow: "implement", + }) + if err != nil { + t.Fatalf("InsertBinding() error = %v", err) + } + + // A draft cannot skip review. + if err := st.ApproveBinding(t.Context(), id); !errors.Is(err, storecontract.ErrBindingTransition) { + t.Fatalf("ApproveBinding(draft) error = %v, want ErrBindingTransition: draft must not skip review", err) + } + + // Editing sends it to pending_approval. + if err := st.UpdateBinding(t.Context(), binding.Binding{ + ID: id, Name: "n2", Matcher: binding.Matcher{Source: "sentry"}, Workflow: "implement", + }); err != nil { + t.Fatalf("UpdateBinding() error = %v", err) + } + b, err := st.GetBinding(t.Context(), id) + if err != nil { + t.Fatalf("GetBinding() error = %v", err) + } + if b.Status != binding.StatusPendingApproval { + t.Fatalf("status after edit = %q, want %q", b.Status, binding.StatusPendingApproval) + } + + // Approval arms it, and only from pending_approval. + if err := st.ApproveBinding(t.Context(), id); err != nil { + t.Fatalf("ApproveBinding(pending_approval) error = %v", err) + } + if b, err = st.GetBinding(t.Context(), id); err != nil || b.Status != binding.StatusArmed { + t.Fatalf("status after approve = %q (err %v), want %q", b.Status, err, binding.StatusArmed) + } + if err := st.ApproveBinding(t.Context(), id); !errors.Is(err, storecontract.ErrBindingTransition) { + t.Errorf("re-approving an armed binding = %v, want ErrBindingTransition", err) + } + + // Only armed bindings dispatch. + armed, err := st.ArmedBindingsForSource(t.Context(), "sentry") + if err != nil || len(armed) != 1 { + t.Fatalf("ArmedBindingsForSource() = %d bindings (err %v), want 1", len(armed), err) + } +} + +// TestOneBindingPerSource pins the overlap guard: a second binding on a +// source that already has one is refused, so a capture can never match two. +func TestOneBindingPerSource(t *testing.T) { + st := newStore(t) + mk := func() error { + _, err := st.InsertBinding(t.Context(), binding.Binding{ + Name: "n", Matcher: binding.Matcher{Source: "sentry"}, Workflow: "implement", + }) + return err + } + if err := mk(); err != nil { + t.Fatalf("first InsertBinding() error = %v", err) + } + if err := mk(); !errors.Is(err, storecontract.ErrBindingOverlap) { + t.Errorf("second InsertBinding() error = %v, want ErrBindingOverlap", err) + } +} diff --git a/internal/infrastructure/edastore/testsupport_test.go b/internal/infrastructure/edastore/testsupport_test.go new file mode 100644 index 000000000..3a817a523 --- /dev/null +++ b/internal/infrastructure/edastore/testsupport_test.go @@ -0,0 +1,30 @@ +package edastore + +import ( + "testing" + + "github.com/samcharles93/archie-core/internal/domain/binding" +) + +// bindingInput is the small shape the cipher tests need, kept separate from +// binding.Binding so a domain field addition does not churn every test. +type bindingInput struct { + Name string + Source string + Workflow string + Secret string +} + +func insertBinding(t *testing.T, s *Store, in bindingInput) string { + t.Helper() + id, err := s.InsertBinding(t.Context(), binding.Binding{ + Name: in.Name, + Matcher: binding.Matcher{Source: in.Source}, + Workflow: in.Workflow, + Secret: in.Secret, + }) + if err != nil { + t.Fatalf("InsertBinding() error = %v", err) + } + return id +} diff --git a/internal/infrastructure/stateadmin/stateadmin.go b/internal/infrastructure/stateadmin/stateadmin.go index 4c8dd6fec..62c31e221 100644 --- a/internal/infrastructure/stateadmin/stateadmin.go +++ b/internal/infrastructure/stateadmin/stateadmin.go @@ -50,6 +50,11 @@ var refusedPrefixes = []string{"/api/backups", "/api/sql"} // collection name is prefixed so it cannot collide with a PocketBase system // collection, and every query must select an "id": PocketBase keys view // records on it, and resources is keyed by kind rather than a row id. +// +// Only the task-lifecycle tables appear here. The event-capture tables moved +// to internal/infrastructure/edastore, which owns them as real, editable +// collections; mirroring them as read-only views would be a second, weaker +// surface over data that already has a better one. var views = []struct { name string query string @@ -57,7 +62,6 @@ var views = []struct { {"archie_tasks", `SELECT id, owner, repo, issue_number, title, status, workflow, stage, branch, pr_number, tokens_used, iterations, attempt, park_reason, identity, created_at, updated_at FROM tasks`}, {"archie_transitions", `SELECT id, task_id, at, from_status, to_status, detail FROM transitions`}, {"archie_events", `SELECT id, at, kind, task_id, repo, issue, workflow, stage, attempt FROM events`}, - {"archie_bindings", `SELECT id, name, source, mapping_id, workflow, owner, repo, version, status FROM bindings`}, {"archie_identities", `SELECT id, kind, display_name, lifecycle, version, created_at, updated_at FROM identities`}, {"archie_resources", `SELECT kind AS id, kind, version, updated_at FROM resources`}, } diff --git a/internal/infrastructure/staterpc/attribution_test.go b/internal/infrastructure/staterpc/attribution_test.go index 68a5ba91b..8535e00b4 100644 --- a/internal/infrastructure/staterpc/attribution_test.go +++ b/internal/infrastructure/staterpc/attribution_test.go @@ -31,7 +31,7 @@ func TestEventAttributionSurvivesTheStateStoreBoundary(t *testing.T) { local := store.OpenTest(t) var c contract = local if mode == "grpc" { - c = remoteTaskStore(t, local, nil) + c = remoteTaskStore(t, local, nil, nil) } task, err := c.EnqueueChatTask(ctx, "acme", "widget", "title", "body", "implement", "") diff --git a/internal/infrastructure/staterpc/capture_stream_test.go b/internal/infrastructure/staterpc/capture_stream_test.go index f2db99e48..2bc059e87 100644 --- a/internal/infrastructure/staterpc/capture_stream_test.go +++ b/internal/infrastructure/staterpc/capture_stream_test.go @@ -5,37 +5,49 @@ import ( "testing" "github.com/samcharles93/archie-core/internal/domain/binding" + "github.com/samcharles93/archie-core/internal/domain/mapping" + "github.com/samcharles93/archie-core/internal/infrastructure/edastore" "github.com/samcharles93/archie-core/internal/store" ) func TestCaptureListsExceedUnaryMessageLimit(t *testing.T) { local := store.OpenTest(t) + eda := edastore.OpenTest(t) body := strings.Repeat("x", 256<<10) for range 20 { - if _, err := local.InsertCapture(t.Context(), store.CapturedEvent{Source: "large", Body: body, Authenticated: true}, 0, 0); err != nil { + if _, err := eda.InsertCapture(t.Context(), store.CapturedEvent{Source: "large", Body: body, Authenticated: true}, 0, 0); err != nil { t.Fatal(err) } } + // A binding's mapping is a real relation now, so it must point at a + // mapping that exists; the store refuses a dangling id. + mappingID, mapErr := eda.InsertMapping(t.Context(), mapping.Mapping{ + Name: "m", + Fields: []mapping.Field{{Name: "title", Path: "title", Type: mapping.TypeString}}, + }) + if mapErr != nil { + t.Fatal(mapErr) + } // ListUndispatchedCaptures only returns sources with an armed binding - // (internal/store/bindings.go), so "large" needs one taken through the + // (edastore.ArmedBindingsForSource), so "large" needs one taken through the // public draft -> pending_approval -> armed lifecycle. - id, err := local.InsertBinding(t.Context(), binding.Binding{ + id, err := eda.InsertBinding(t.Context(), binding.Binding{ Name: "large binding", Matcher: binding.Matcher{Source: "large"}, - MappingID: 1, Workflow: "implement", Secret: "0123456789abcdef0123456789abcdef", + MappingID: mappingID, Workflow: "implement", Secret: "0123456789abcdef0123456789abcdef", }) if err != nil { t.Fatal(err) } - if err := local.UpdateBinding(t.Context(), binding.Binding{ + if err := eda.UpdateBinding(t.Context(), binding.Binding{ ID: id, Name: "large binding", Matcher: binding.Matcher{Source: "large"}, - MappingID: 1, Workflow: "implement", Secret: "0123456789abcdef0123456789abcdef", + MappingID: mappingID, Workflow: "implement", Secret: "0123456789abcdef0123456789abcdef", }); err != nil { t.Fatal(err) } - if err := local.ApproveBinding(t.Context(), id); err != nil { + if err := eda.ApproveBinding(t.Context(), id); err != nil { t.Fatal(err) } - remote := remoteContract(t, local) + remote := remoteEDA(t, local, eda) for _, undispatched := range []bool{false, true} { var captures []store.CapturedEvent var err error diff --git a/internal/infrastructure/staterpc/client.go b/internal/infrastructure/staterpc/client.go index 57a23cb0c..3037918e8 100644 --- a/internal/infrastructure/staterpc/client.go +++ b/internal/infrastructure/staterpc/client.go @@ -257,10 +257,10 @@ func (c *Client) TokensByDay(ctx context.Context, days int) ([]storecontract.Day // Capture -func (c *Client) InsertCapture(ctx context.Context, ce storecontract.CapturedEvent, retention time.Duration, maxEvents int) (int64, error) { +func (c *Client) InsertCapture(ctx context.Context, ce storecontract.CapturedEvent, retention time.Duration, maxEvents int) (string, error) { r, err := c.client.InsertCapture(ctx, &pb.InsertCaptureRequest{Capture: capturedEventProto(ce), RetentionSeconds: int64(retention.Seconds()), MaxEvents: int64(maxEvents)}) if err != nil { - return 0, unmapError(err) + return "", unmapError(err) } return r.Id, nil } @@ -357,15 +357,15 @@ func (c *Client) ListCaptures(ctx context.Context, limit int) ([]storecontract.C // Mapping -func (c *Client) InsertMapping(ctx context.Context, m mapping.Mapping) (int64, error) { +func (c *Client) InsertMapping(ctx context.Context, m mapping.Mapping) (string, error) { r, err := c.client.InsertMapping(ctx, &pb.InsertMappingRequest{Mapping: mappingProto(m)}) if err != nil { - return 0, unmapError(err) + return "", unmapError(err) } return r.Id, nil } -func (c *Client) GetMapping(ctx context.Context, id int64) (*mapping.Mapping, error) { +func (c *Client) GetMapping(ctx context.Context, id string) (*mapping.Mapping, error) { r, err := c.client.GetMapping(ctx, &pb.GetMappingRequest{Id: id}) if err != nil { return nil, unmapError(err) @@ -390,22 +390,22 @@ func (c *Client) UpdateMapping(ctx context.Context, m mapping.Mapping) error { return unmapError(err) } -func (c *Client) DeleteMapping(ctx context.Context, id int64) error { +func (c *Client) DeleteMapping(ctx context.Context, id string) error { _, err := c.client.DeleteMapping(ctx, &pb.DeleteMappingRequest{Id: id}) return unmapError(err) } // Binding -func (c *Client) InsertBinding(ctx context.Context, b binding.Binding) (int64, error) { +func (c *Client) InsertBinding(ctx context.Context, b binding.Binding) (string, error) { r, err := c.client.InsertBinding(ctx, &pb.InsertBindingRequest{Binding: bindingProto(b)}) if err != nil { - return 0, unmapError(err) + return "", unmapError(err) } return r.Id, nil } -func (c *Client) GetBinding(ctx context.Context, id int64) (*binding.Binding, error) { +func (c *Client) GetBinding(ctx context.Context, id string) (*binding.Binding, error) { r, err := c.client.GetBinding(ctx, &pb.GetBindingRequest{Id: id}) if err != nil { return nil, unmapError(err) @@ -430,12 +430,12 @@ func (c *Client) UpdateBinding(ctx context.Context, b binding.Binding) error { return unmapError(err) } -func (c *Client) DeleteBinding(ctx context.Context, id int64) error { +func (c *Client) DeleteBinding(ctx context.Context, id string) error { _, err := c.client.DeleteBinding(ctx, &pb.DeleteBindingRequest{Id: id}) return unmapError(err) } -func (c *Client) ApproveBinding(ctx context.Context, id int64) error { +func (c *Client) ApproveBinding(ctx context.Context, id string) error { _, err := c.client.ApproveBinding(ctx, &pb.ApproveBindingRequest{Id: id}) return unmapError(err) } @@ -450,7 +450,7 @@ func (c *Client) ArmedBindingsForSource(ctx context.Context, source string) ([]b return mapValues(r.Bindings, bindingValue), nil } -func (c *Client) RecordDispatch(ctx context.Context, bindingID, bindingVersion, captureID, taskID int64) error { +func (c *Client) RecordDispatch(ctx context.Context, bindingID string, bindingVersion int64, captureID string, taskID int64) error { _, err := c.client.RecordDispatch(ctx, &pb.RecordDispatchRequest{BindingId: bindingID, BindingVersion: bindingVersion, CaptureId: captureID, TaskId: taskID}) return unmapError(err) } @@ -489,7 +489,7 @@ func (c *Client) ListUndispatchedCaptures(ctx context.Context, sources []string, // BindingTaskCreator -func (c *Client) EnqueueBindingTask(ctx context.Context, owner, repo, title, body, wf, identity string, bindingID int64, bindingVersion int) (*task.Task, error) { +func (c *Client) EnqueueBindingTask(ctx context.Context, owner, repo, title, body, wf, identity, bindingID string, bindingVersion int) (*task.Task, error) { r, err := c.client.EnqueueBindingTask(ctx, &pb.EnqueueBindingTaskRequest{Owner: owner, Repo: repo, Title: title, Body: body, Workflow: wf, Identity: identity, BindingId: bindingID, BindingVersion: int64(bindingVersion)}) if err != nil { return nil, unmapError(err) diff --git a/internal/infrastructure/staterpc/conformance_test.go b/internal/infrastructure/staterpc/conformance_test.go index 78b9ae8a5..b2a322248 100644 --- a/internal/infrastructure/staterpc/conformance_test.go +++ b/internal/infrastructure/staterpc/conformance_test.go @@ -14,6 +14,7 @@ import ( "github.com/samcharles93/archie-core/internal/domain/binding" "github.com/samcharles93/archie-core/internal/domain/mapping" "github.com/samcharles93/archie-core/internal/events" + "github.com/samcharles93/archie-core/internal/infrastructure/edastore" "github.com/samcharles93/archie-core/internal/store" ) @@ -24,13 +25,19 @@ import ( // docs/prds/state-store-contract.md §11's conformance requirement. type contract interface { store.TaskStore + store.BindingTaskCreator + store.ConfigSnapshotStore + store.ApplyStatusStore +} + +// edaContract is the event-capture half, which the PocketBase store serves. +// It is a separate interface because no single type implements both halves +// any more: that split is the point of the migration, not an accident. +type edaContract interface { store.CaptureStore store.MappingStore store.BindingStore store.BindingDispatcher - store.BindingTaskCreator - store.ConfigSnapshotStore - store.ApplyStatusStore } // playbookContract is the playbook-dispatch idempotency-ledger surface, split @@ -48,24 +55,43 @@ type taskLogContract interface { store.TaskLogStore } +// remoteEDA fronts both stores for a battery that exercises event-capture +// surfaces over the wire. +func remoteEDA(t *testing.T, local *store.Store, eda *edastore.Store) *Client { + t.Helper() + return remoteTaskStore(t, local, nil, eda) +} + func remoteContract(t *testing.T, local *store.Store) contract { t.Helper() - return remoteTaskStore(t, local, nil) + return remoteTaskStore(t, local, nil, nil) } // remoteTaskStore serves local behind a bufconn listener with a task-log // reader attached, and returns a client for it. The reader is a parameter // because the log files are internal/logging's, not the store's: this contract // fronts a reader rather than a table. -func remoteTaskStore(t *testing.T, local *store.Store, logs store.TaskLogStore) *Client { +// remoteTaskStore fronts the same two stores the real composition wires: the +// task store for the task surfaces, the event-capture store for captures, +// mappings, bindings and the dispatch ledgers. eda may be nil for a battery +// that exercises only task surfaces. +func remoteTaskStore(t *testing.T, local *store.Store, logs store.TaskLogStore, eda *edastore.Store) *Client { t.Helper() listener := bufconn.Listen(1 << 20) server := grpc.NewServer() - RegisterServer(server, Deps{ - Tasks: local, Captures: local, Mappings: local, Bindings: local, - BindingDispatcher: local, BindingTaskCreator: local, PlaybookDispatcher: local, ConfigSnapshots: local, ApplyStatus: local, + deps := Deps{ + Tasks: local, BindingTaskCreator: local, + ConfigSnapshots: local, ApplyStatus: local, TaskLogs: logs, - }) + } + if eda != nil { + deps.Captures = eda + deps.Mappings = eda + deps.Bindings = eda + deps.BindingDispatcher = eda + deps.PlaybookDispatcher = eda + } + RegisterServer(server, deps) go func() { _ = server.Serve(listener) }() t.Cleanup(func() { server.Stop(); _ = listener.Close() }) conn, err := grpc.NewClient("passthrough:///state", @@ -87,11 +113,14 @@ func TestStateStoreConformance(t *testing.T) { t.Run(mode, func(t *testing.T) { ctx := t.Context() local := store.OpenTest(t) + eda := edastore.OpenTest(t) var c contract = local - var pc playbookContract = local + var ec edaContract = eda + var pc playbookContract = eda if mode == "grpc" { - remote := remoteTaskStore(t, local, nil) + remote := remoteTaskStore(t, local, nil, eda) c = remote + ec = remote pc = remote } @@ -253,30 +282,30 @@ func TestStateStoreConformance(t *testing.T) { } // Mapping: found=false, ErrMappingNotFound. - mappingID, err := c.InsertMapping(ctx, mapping.Mapping{Name: "m1", Fields: []mapping.Field{{Name: "a", Path: "a", Type: mapping.TypeString}}}) - if err != nil || mappingID == 0 { + mappingID, err := ec.InsertMapping(ctx, mapping.Mapping{Name: "m1", Fields: []mapping.Field{{Name: "a", Path: "a", Type: mapping.TypeString}}}) + if err != nil || mappingID == "" { t.Fatalf("InsertMapping: %v %v", mappingID, err) } - missingMapping, err := c.GetMapping(ctx, mappingID+999999) + missingMapping, err := ec.GetMapping(ctx, "rabsent00000000") if err != nil || missingMapping != nil { t.Fatalf("GetMapping missing: %+v %v", missingMapping, err) } - gotMapping, err := c.GetMapping(ctx, mappingID) + gotMapping, err := ec.GetMapping(ctx, mappingID) if err != nil || gotMapping == nil || gotMapping.Name != "m1" { t.Fatalf("GetMapping: %+v %v", gotMapping, err) } - if _, err := c.ListMappings(ctx); err != nil { + if _, err := ec.ListMappings(ctx); err != nil { t.Fatalf("ListMappings: %v", err) } gotMapping.Name = "m1-renamed" - if err := c.UpdateMapping(ctx, *gotMapping); err != nil { + if err := ec.UpdateMapping(ctx, *gotMapping); err != nil { t.Fatalf("UpdateMapping: %v", err) } - err = c.UpdateMapping(ctx, mapping.Mapping{ID: mappingID + 999999, Name: "x", Fields: gotMapping.Fields}) + err = ec.UpdateMapping(ctx, mapping.Mapping{ID: "rabsent00000000", Name: "x", Fields: gotMapping.Fields}) if !errors.Is(err, store.ErrMappingNotFound) { t.Fatalf("UpdateMapping missing = %v, want ErrMappingNotFound", err) } - err = c.DeleteMapping(ctx, mappingID+999999) + err = ec.DeleteMapping(ctx, "rabsent00000000") if !errors.Is(err, store.ErrMappingNotFound) { t.Fatalf("DeleteMapping missing = %v, want ErrMappingNotFound", err) } @@ -284,63 +313,63 @@ func TestStateStoreConformance(t *testing.T) { // Binding: found=false, ErrBindingNotFound, ErrBindingOverlap, // ErrBindingTransition, and the dispatch surface incl. // ErrAlreadyDispatched. - bindingID, err := c.InsertBinding(ctx, binding.Binding{ + bindingID, err := ec.InsertBinding(ctx, binding.Binding{ Name: "b1", Matcher: binding.Matcher{Source: "sentry"}, MappingID: mappingID, Workflow: "implement", Secret: "0123456789abcdef0123456789abcdef", }) - if err != nil || bindingID == 0 { + if err != nil || bindingID == "" { t.Fatalf("InsertBinding: %v %v", bindingID, err) } - _, err = c.InsertBinding(ctx, binding.Binding{ + _, err = ec.InsertBinding(ctx, binding.Binding{ Name: "b2", Matcher: binding.Matcher{Source: "sentry"}, MappingID: mappingID, Workflow: "implement", Secret: "0123456789abcdef0123456789abcdef", }) if !errors.Is(err, store.ErrBindingOverlap) { t.Fatalf("InsertBinding overlap = %v, want ErrBindingOverlap", err) } - missingBinding, err := c.GetBinding(ctx, bindingID+999999) + missingBinding, err := ec.GetBinding(ctx, "rabsent00000000") if err != nil || missingBinding != nil { t.Fatalf("GetBinding missing: %+v %v", missingBinding, err) } - gotBinding, err := c.GetBinding(ctx, bindingID) + gotBinding, err := ec.GetBinding(ctx, bindingID) if err != nil || gotBinding == nil || gotBinding.Name != "b1" { t.Fatalf("GetBinding: %+v %v", gotBinding, err) } - if _, err := c.ListBindings(ctx); err != nil { + if _, err := ec.ListBindings(ctx); err != nil { t.Fatalf("ListBindings: %v", err) } - err = c.ApproveBinding(ctx, bindingID) + err = ec.ApproveBinding(ctx, bindingID) if !errors.Is(err, store.ErrBindingTransition) { t.Fatalf("ApproveBinding from draft = %v, want ErrBindingTransition", err) } - err = c.DeleteBinding(ctx, bindingID+999999) + err = ec.DeleteBinding(ctx, "rabsent00000000") if !errors.Is(err, store.ErrBindingNotFound) { t.Fatalf("DeleteBinding missing = %v, want ErrBindingNotFound", err) } // Capture + dispatch. - captureID, err := c.InsertCapture(ctx, store.CapturedEvent{Source: "sentry", Body: `{"id":1}`, Authenticated: true}, 0, 0) - if err != nil || captureID == 0 { + captureID, err := ec.InsertCapture(ctx, store.CapturedEvent{Source: "sentry", Body: `{"id":1}`, Authenticated: true}, 0, 0) + if err != nil || captureID == "" { t.Fatalf("InsertCapture: %v %v", captureID, err) } - if _, err := c.ListCaptures(ctx, 10); err != nil { + if _, err := ec.ListCaptures(ctx, 10); err != nil { t.Fatalf("ListCaptures: %v", err) } - if _, err := c.ArmedBindingsForSource(ctx, "sentry"); err != nil { + if _, err := ec.ArmedBindingsForSource(ctx, "sentry"); err != nil { t.Fatalf("ArmedBindingsForSource: %v", err) } bindingTask, err := c.EnqueueBindingTask(ctx, "acme", "widget", "t", "b", "implement", "", bindingID, 1) if err != nil || bindingTask == nil { t.Fatalf("EnqueueBindingTask: %+v %v", bindingTask, err) } - if err := c.RecordDispatch(ctx, bindingID, 1, captureID, bindingTask.ID); err != nil { + if err := ec.RecordDispatch(ctx, bindingID, 1, captureID, bindingTask.ID); err != nil { t.Fatalf("RecordDispatch: %v", err) } - err = c.RecordDispatch(ctx, bindingID, 1, captureID, bindingTask.ID) + err = ec.RecordDispatch(ctx, bindingID, 1, captureID, bindingTask.ID) if !errors.Is(err, store.ErrAlreadyDispatched) { t.Fatalf("RecordDispatch dup = %v, want ErrAlreadyDispatched", err) } - if _, err := c.ListUndispatchedCaptures(ctx, []string{"sentry"}, 10); err != nil { + if _, err := ec.ListUndispatchedCaptures(ctx, []string{"sentry"}, 10); err != nil { t.Fatalf("ListUndispatchedCaptures: %v", err) } diff --git a/internal/infrastructure/staterpc/task_grants_test.go b/internal/infrastructure/staterpc/task_grants_test.go index fac2710aa..46bbf252e 100644 --- a/internal/infrastructure/staterpc/task_grants_test.go +++ b/internal/infrastructure/staterpc/task_grants_test.go @@ -13,8 +13,10 @@ import ( "google.golang.org/grpc/test/bufconn" "github.com/samcharles93/archie-core/internal/domain/binding" + "github.com/samcharles93/archie-core/internal/domain/mapping" "github.com/samcharles93/archie-core/internal/domain/workflow" "github.com/samcharles93/archie-core/internal/events" + "github.com/samcharles93/archie-core/internal/infrastructure/edastore" "github.com/samcharles93/archie-core/internal/store" ) @@ -31,7 +33,15 @@ func grantsServer(t *testing.T, adminToken string) (grants *TaskGrants, dial fun grpc.ChainUnaryInterceptor(grants.UnaryInterceptor(adminToken)), grpc.ChainStreamInterceptor(grants.StreamInterceptor(adminToken)), ) - RegisterServer(server, Deps{Tasks: local, Captures: local, Bindings: local, BindingDispatcher: local, PlaybookDispatcher: local, Grants: grants, ConfigSnapshots: local, ApplyStatus: local}) + // One event-capture store serves every EDA surface, as the real + // composition does: separate instances would not share a database. + eda := edastore.OpenTest(t) + RegisterServer(server, Deps{ + Tasks: local, ConfigSnapshots: local, ApplyStatus: local, + Captures: eda, Mappings: eda, Bindings: eda, + BindingDispatcher: eda, PlaybookDispatcher: eda, + Grants: grants, + }) go func() { _ = server.Serve(listener) }() t.Cleanup(func() { server.Stop(); _ = listener.Close() }) @@ -74,18 +84,27 @@ func TestCaptureStreamsCarryTheAdminToken(t *testing.T) { } } // ListUndispatchedCaptures only returns sources with an armed binding - // (internal/store/bindings.go), so "large" needs one taken through the + // A binding's mapping is a real relation now, so it must point at a + // mapping that exists; the store refuses a dangling id. + mappingID, mapErr := admin.InsertMapping(t.Context(), mapping.Mapping{ + Name: "m", + Fields: []mapping.Field{{Name: "title", Path: "title", Type: mapping.TypeString}}, + }) + if mapErr != nil { + t.Fatal(mapErr) + } + // (edastore.ArmedBindingsForSource), so "large" needs one taken through the // public draft -> pending_approval -> armed lifecycle. id, err := admin.InsertBinding(ctx, binding.Binding{ Name: "large binding", Matcher: binding.Matcher{Source: "large"}, - MappingID: 1, Workflow: "implement", Secret: "0123456789abcdef0123456789abcdef", + MappingID: mappingID, Workflow: "implement", Secret: "0123456789abcdef0123456789abcdef", }) if err != nil { t.Fatal(err) } if err := admin.UpdateBinding(ctx, binding.Binding{ ID: id, Name: "large binding", Matcher: binding.Matcher{Source: "large"}, - MappingID: 1, Workflow: "implement", Secret: "0123456789abcdef0123456789abcdef", + MappingID: mappingID, Workflow: "implement", Secret: "0123456789abcdef0123456789abcdef", }); err != nil { t.Fatal(err) } diff --git a/internal/infrastructure/staterpc/task_log_nilslice_test.go b/internal/infrastructure/staterpc/task_log_nilslice_test.go index 8837f9a76..fcef50e59 100644 --- a/internal/infrastructure/staterpc/task_log_nilslice_test.go +++ b/internal/infrastructure/staterpc/task_log_nilslice_test.go @@ -54,7 +54,7 @@ func TestTaskLogPageCarriesNoNilCollectionFields(t *testing.T) { var c taskLogContract = reader if mode == "grpc" { - c = remoteTaskStore(t, store.OpenTest(t), reader) + c = remoteTaskStore(t, store.OpenTest(t), reader, nil) } page, err := c.TaskLog(ctx, taskID, 0, logging.Query{}) diff --git a/internal/infrastructure/staterpc/task_log_test.go b/internal/infrastructure/staterpc/task_log_test.go index f641eb1b0..c87185888 100644 --- a/internal/infrastructure/staterpc/task_log_test.go +++ b/internal/infrastructure/staterpc/task_log_test.go @@ -41,7 +41,7 @@ func TestTaskLogContract(t *testing.T) { var c taskLogContract = reader if mode == "grpc" { - c = remoteTaskStore(t, store.OpenTest(t), reader) + c = remoteTaskStore(t, store.OpenTest(t), reader, nil) } t.Run("a page carries the decoded entries and their fields", func(t *testing.T) { diff --git a/internal/store/bindings.go b/internal/store/bindings.go deleted file mode 100644 index d04c49d95..000000000 --- a/internal/store/bindings.go +++ /dev/null @@ -1,450 +0,0 @@ -package store - -import ( - "context" - "database/sql" - "errors" - "fmt" - "strings" - "time" - - "github.com/samcharles93/archie-core/internal/domain/binding" -) - -const bindingsSchema = ` -CREATE TABLE IF NOT EXISTS bindings ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT NOT NULL DEFAULT '', - source TEXT NOT NULL DEFAULT '', - mapping_id INTEGER NOT NULL DEFAULT 0, - workflow TEXT NOT NULL DEFAULT '', - owner TEXT NOT NULL DEFAULT '', - repo TEXT NOT NULL DEFAULT '', - version INTEGER NOT NULL DEFAULT 1, - status TEXT NOT NULL DEFAULT 'draft', - secret TEXT NOT NULL DEFAULT '', - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL, - CHECK (status IN ('draft','pending_approval','armed')) -); -CREATE INDEX IF NOT EXISTS idx_bindings_source_status ON bindings(source, status); -CREATE TABLE IF NOT EXISTS binding_dispatches ( - binding_id INTEGER NOT NULL, - binding_version INTEGER NOT NULL, - capture_id INTEGER NOT NULL, - task_id INTEGER NOT NULL, - dispatched_at TEXT NOT NULL, - PRIMARY KEY (binding_id, capture_id) -); -` - -// ErrBindingNotFound is returned by UpdateBinding and DeleteBinding when no -// row matches the given ID. GetBinding uses (nil, nil) instead, matching -// TaskByID's convention -- "not found" is the normal answer to "does a -// binding with this ID currently exist," not an error. - -// ErrBindingOverlap is returned by InsertBinding and UpdateBinding when a -// different binding already covers the same source. Two bindings for one -// source would race over the same inbound webhook; the store refuses rather -// than picking an arbitrary winner. - -// ErrBindingTransition is returned by ApproveBinding when the from-status -// is anything other than pending_approval. Approve is the only sanctioned -// transition into armed, and the guard is the SQL WHERE clause, not a -// separate read. - -// ErrAlreadyDispatched is returned by RecordDispatch when the (binding_id, -// capture_id) pair already exists in binding_dispatches, and by -// RecordPlaybookDispatch when the (playbook_id, playbook_version, event_id, -// action_id) tuple already exists in playbook_dispatches. Each dedup ledger -// is an at-most-once guarantee; the caller is expected to surface this to the -// dispatch loop rather than retry. - -// bindingTimeLayout follows captureTimeLayout's reasoning: fixed-width -// RFC3339 so string and chronological order agree. -const bindingTimeLayout = time.RFC3339 - -// InsertBinding persists a new binding in status=draft regardless of the -// caller's Status field, stamps CreatedAt/UpdatedAt to now, and seeds -// Version=1. A pre-insert overlap check refuses a duplicate source in the -// same transaction; ErrBindingOverlap is returned rather than a silent -// constraint violation. -func (s *Store) InsertBinding(ctx context.Context, b binding.Binding) (int64, error) { - tx, err := s.db.BeginTx(ctx, nil) - if err != nil { - return 0, err - } - defer func() { _ = tx.Rollback() }() - - var overlapID int64 - row := tx.QueryRowContext(ctx, `SELECT id FROM bindings WHERE source = ? LIMIT 1`, b.Matcher.Source) - if err := row.Scan(&overlapID); err == nil { - return 0, ErrBindingOverlap - } else if !errors.Is(err, sql.ErrNoRows) { - return 0, err - } - - now := time.Now().UTC().Format(bindingTimeLayout) - secret, err := s.encryptBindingSecret(b.Secret) - if err != nil { - return 0, err - } - res, err := tx.ExecContext(ctx, ` - INSERT INTO bindings (name, source, mapping_id, workflow, owner, repo, version, status, secret, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, 1, 'draft', ?, ?, ?)`, - b.Name, b.Matcher.Source, b.MappingID, b.Workflow, b.Owner, b.Repo, secret, now, now) - if err != nil { - return 0, err - } - id, err := res.LastInsertId() - if err != nil { - return 0, err - } - if err := tx.Commit(); err != nil { - return 0, err - } - return id, nil -} - -// GetBinding returns the binding with the given ID, or (nil, nil) if none -// exists. -func (s *Store) GetBinding(ctx context.Context, id int64) (*binding.Binding, error) { - row := s.db.QueryRowContext(ctx, ` - SELECT id, name, source, mapping_id, workflow, owner, repo, version, status, secret, created_at, updated_at - FROM bindings WHERE id=?`, id) - b, err := scanBindingRow(row) - if errors.Is(err, sql.ErrNoRows) { - return nil, nil - } - if err != nil { - return nil, err - } - if err := s.decryptBindingSecret(b); err != nil { - return nil, err - } - return b, nil -} - -// ListBindings returns every binding, newest first. -func (s *Store) ListBindings(ctx context.Context) (out []binding.Binding, retErr error) { - rows, err := s.db.QueryContext(ctx, ` - SELECT id, name, source, mapping_id, workflow, owner, repo, version, status, secret, created_at, updated_at - FROM bindings ORDER BY id DESC`) - if err != nil { - return nil, err - } - defer func() { retErr = errors.Join(retErr, rows.Close()) }() - - for rows.Next() { - b, err := scanBindingRow(rows) - if err != nil { - return nil, err - } - if err := s.decryptBindingSecret(b); err != nil { - return nil, err - } - out = append(out, *b) - } - return out, rows.Err() -} - -// UpdateBinding replaces the editable fields of an existing binding and -// bumps Version by 1. Per docs/prds/webhook-intake-security.md point 2: -// any edit always drops status to pending_approval, including edits to -// already-armed bindings -- an edit cannot silently re-arm, or the -// approval gate is decorative. Secret is preserved when the caller passes -// an empty string via COALESCE(NULLIF(?, ”), secret) so a partial update -// cannot erase the HMAC secret. Returns ErrBindingNotFound when no row -// matches, ErrBindingOverlap when the new source collides with a different -// binding. -func (s *Store) UpdateBinding(ctx context.Context, b binding.Binding) error { - tx, err := s.db.BeginTx(ctx, nil) - if err != nil { - return err - } - defer func() { _ = tx.Rollback() }() - - var overlapID int64 - row := tx.QueryRowContext(ctx, `SELECT id FROM bindings WHERE source = ? AND id != ? LIMIT 1`, - b.Matcher.Source, b.ID) - if err := row.Scan(&overlapID); err == nil { - return ErrBindingOverlap - } else if !errors.Is(err, sql.ErrNoRows) { - return err - } - - now := time.Now().UTC().Format(bindingTimeLayout) - secret, err := s.encryptBindingSecret(b.Secret) - if err != nil { - return err - } - res, err := tx.ExecContext(ctx, ` - UPDATE bindings SET name=?, source=?, mapping_id=?, workflow=?, owner=?, repo=?, secret=COALESCE(NULLIF(?, ''), secret), - status = 'pending_approval', - version = version + 1, - updated_at = ? - WHERE id=?`, - b.Name, b.Matcher.Source, b.MappingID, b.Workflow, b.Owner, b.Repo, secret, now, b.ID) - if err != nil { - return err - } - n, err := res.RowsAffected() - if err != nil { - return err - } - if n == 0 { - return ErrBindingNotFound - } - return tx.Commit() -} - -// DeleteBinding removes a binding by ID along with any rows it owns in -// binding_dispatches. The dedup rows are deleted in the same transaction -// so a recreate with the same source does not inherit old dedup state. -// Returns ErrBindingNotFound if it does not exist. -func (s *Store) DeleteBinding(ctx context.Context, id int64) error { - tx, err := s.db.BeginTx(ctx, nil) - if err != nil { - return err - } - defer func() { _ = tx.Rollback() }() - - if _, err := tx.ExecContext(ctx, `DELETE FROM binding_dispatches WHERE binding_id = ?`, id); err != nil { - return err - } - res, err := tx.ExecContext(ctx, `DELETE FROM bindings WHERE id=?`, id) - if err != nil { - return err - } - n, err := res.RowsAffected() - if err != nil { - return err - } - if n == 0 { - return ErrBindingNotFound - } - return tx.Commit() -} - -// ApproveBinding moves a binding from pending_approval to armed. The -// status guard is the SQL WHERE clause: a row whose status does not match -// pending_approval yields RowsAffected == 0 and is distinguished as a -// transition error (row exists, wrong from-status) or a not-found error -// (no row) by a follow-up SELECT inside the same transaction. Approve is -// also refused when a different binding is already armed for the same -// source: the overlap rule has to hold across status changes, so this is -// a TOCTOU guard rather than a primary check (Insert/Update own the -// initial overlap prevention). -func (s *Store) ApproveBinding(ctx context.Context, id int64) error { - tx, err := s.db.BeginTx(ctx, nil) - if err != nil { - return err - } - defer func() { _ = tx.Rollback() }() - - var source string - row := tx.QueryRowContext(ctx, `SELECT source FROM bindings WHERE id=?`, id) - if err := row.Scan(&source); err != nil { - if errors.Is(err, sql.ErrNoRows) { - return ErrBindingNotFound - } - return err - } - - var otherArmed int64 - row = tx.QueryRowContext(ctx, - `SELECT id FROM bindings WHERE source=? AND status='armed' AND id != ? LIMIT 1`, - source, id) - if err := row.Scan(&otherArmed); err == nil { - return ErrBindingOverlap - } else if !errors.Is(err, sql.ErrNoRows) { - return err - } - - now := time.Now().UTC().Format(bindingTimeLayout) - res, err := tx.ExecContext(ctx, ` - UPDATE bindings SET status='armed', updated_at=? - WHERE id=? AND status='pending_approval'`, - now, id) - if err != nil { - return err - } - n, err := res.RowsAffected() - if err != nil { - return err - } - if n == 0 { - return ErrBindingTransition - } - return tx.Commit() -} - -// ArmedBindingsForSource returns every armed binding for the given source, -// ordered by id so captureintake's receiver picks a stable winner if multiple -// rows are ever present (the overlap guard should make this impossible in -// practice, but the store returns the full set rather than collapsing). -func (s *Store) ArmedBindingsForSource(ctx context.Context, source string) ([]binding.Binding, error) { - rows, err := s.db.QueryContext(ctx, ` - SELECT id, name, source, mapping_id, workflow, owner, repo, version, status, secret, created_at, updated_at - FROM bindings WHERE source=? AND status='armed' ORDER BY id`, source) - if err != nil { - return nil, err - } - defer func() { _ = rows.Close() }() - - var out []binding.Binding - for rows.Next() { - b, err := scanBindingRow(rows) - if err != nil { - return nil, err - } - if err := s.decryptBindingSecret(b); err != nil { - return nil, err - } - out = append(out, *b) - } - return out, rows.Err() -} - -// encryptBindingSecret seals a plaintext secret for persistence. With no -// cipher configured it returns the value unchanged (legacy plaintext). -// -// An empty secret is the store's sentinel for "no secret" and is returned -// unchanged even when a cipher is installed. Sealing it would yield a -// non-empty envelope, and callers rely on emptiness surviving to the SQL -// layer: UpdateBinding preserves an existing secret through -// COALESCE(NULLIF(?, ”), secret), which cannot fire once encryption has -// turned an empty caller secret into a non-empty value. A partial edit would -// then overwrite the binding's HMAC secret with an encrypted empty string. -func (s *Store) encryptBindingSecret(plaintext string) (string, error) { - if s.bindingsCipher == nil { - return plaintext, nil - } - if plaintext == "" { - return "", nil - } - return s.bindingsCipher.Encrypt(plaintext) -} - -// decryptBindingSecret returns a binding's secret as plaintext. With no cipher -// configured the stored value is returned unchanged. A value that is not an -// encryption envelope is treated as a legacy plaintext row written before a -// cipher was configured and left readable as-is -- encryption applies to rows -// written or edited after the key is installed, not retroactively (see -// docs/prds/binding-secret-encryption.md). -func (s *Store) decryptBindingSecret(b *binding.Binding) error { - if s.bindingsCipher == nil { - return nil - } - if b.Secret == "" || !strings.HasPrefix(b.Secret, bindingEnvelopeMarker+":") { - return nil - } - decrypted, err := s.bindingsCipher.Decrypt(b.Secret) - if err != nil { - return err - } - b.Secret = decrypted - return nil -} - -// RecordDispatch writes a (binding, capture) dedup row using the store's -// own connection (best-effort, not atomic with the task row -- the -// at-most-once guarantee is the INSERT OR IGNORE dedup row itself, which -// is already transactionless in production). INSERT OR IGNORE + -// RowsAffected is the dedup test: a duplicate (binding_id, capture_id) -// is a no-op write and returns ErrAlreadyDispatched rather than a -// constraint error. -func (s *Store) RecordDispatch( - ctx context.Context, - bindingID int64, - bindingVersion int64, - captureID int64, - taskID int64, -) error { - res, err := s.db.ExecContext(ctx, ` - INSERT OR IGNORE INTO binding_dispatches (binding_id, binding_version, capture_id, task_id, dispatched_at) - VALUES (?, ?, ?, ?, ?)`, - bindingID, bindingVersion, captureID, taskID, - time.Now().UTC().Format(bindingTimeLayout)) - if err != nil { - return err - } - n, err := res.RowsAffected() - if err != nil { - return err - } - if n == 0 { - return ErrAlreadyDispatched - } - return nil -} - -// ListUndispatchedCaptures returns authenticated captures that have not -// yet been dispatched by any armed binding whose source matches. The -// nested source query means the result is naturally limited to sources -// with an armed binding; an empty sources slice is therefore a no-arms -// short-circuit and returns (nil, nil) without hitting the table. -func (s *Store) ListUndispatchedCaptures(ctx context.Context, sources []string, limit int) ([]CapturedEvent, error) { - if len(sources) == 0 { - return nil, nil - } - rows, err := s.db.QueryContext(ctx, ` - SELECT id, received_at, source, remote_addr, content_type, headers, body, authenticated - FROM captured_events - WHERE authenticated = 1 - AND id NOT IN (SELECT capture_id FROM binding_dispatches) - AND source IN ( - SELECT source FROM bindings WHERE status = 'armed' GROUP BY source - ) - ORDER BY id - LIMIT ?`, limit) - if err != nil { - return nil, err - } - defer func() { _ = rows.Close() }() - - var out []CapturedEvent - for rows.Next() { - var c CapturedEvent - var receivedAt string - var authenticated int - if err := rows.Scan(&c.ID, &receivedAt, &c.Source, &c.RemoteAddr, - &c.ContentType, &c.Headers, &c.Body, &authenticated); err != nil { - return nil, err - } - parsed, err := parseCaptureTime(receivedAt) - if err != nil { - return nil, err - } - c.ReceivedAt = parsed - c.Authenticated = authenticated != 0 - out = append(out, c) - } - return out, rows.Err() -} - -// bindingScanner is satisfied by both *sql.Row and *sql.Rows so -// scanBindingRow's column list stays in exactly one place. -type bindingScanner interface { - Scan(dest ...any) error -} - -func scanBindingRow(row bindingScanner) (*binding.Binding, error) { - var b binding.Binding - var createdAt, updatedAt string - if err := row.Scan(&b.ID, &b.Name, &b.Matcher.Source, &b.MappingID, &b.Workflow, - &b.Owner, &b.Repo, &b.Version, &b.Status, &b.Secret, &createdAt, &updatedAt); err != nil { - return nil, err - } - created, err := time.Parse(bindingTimeLayout, createdAt) - if err != nil { - return nil, fmt.Errorf("store: unrecognised binding created_at %q: %w", createdAt, err) - } - updated, err := time.Parse(bindingTimeLayout, updatedAt) - if err != nil { - return nil, fmt.Errorf("store: unrecognised binding updated_at %q: %w", updatedAt, err) - } - b.CreatedAt = created.UTC() - b.UpdatedAt = updated.UTC() - return &b, nil -} diff --git a/internal/store/bindings_test.go b/internal/store/bindings_test.go deleted file mode 100644 index c3542e961..000000000 --- a/internal/store/bindings_test.go +++ /dev/null @@ -1,544 +0,0 @@ -package store - -import ( - "database/sql" - "errors" - "path/filepath" - "strings" - "testing" - - "github.com/samcharles93/archie-core/internal/domain/binding" -) - -// testBinding returns a valid binding with a sufficiently long secret for -// Validate. Tests that need a different shape build their own. -func testBinding(source string) binding.Binding { - return binding.Binding{ - Name: "sentry binding", - Matcher: binding.Matcher{Source: source}, - MappingID: 1, - Workflow: "implement", - Status: binding.StatusDraft, - Secret: "0123456789abcdef0123456789abcdef", - } -} - -func TestInsertBindingRoundTrips(t *testing.T) { - s := openTest(t) - b := testBinding("sentry") - id, err := s.InsertBinding(t.Context(), b) - if err != nil { - t.Fatalf("InsertBinding: %v", err) - } - if id == 0 { - t.Fatalf("InsertBinding returned zero id") - } - - got, err := s.GetBinding(t.Context(), id) - if err != nil { - t.Fatalf("GetBinding: %v", err) - } - if got == nil { - t.Fatalf("GetBinding returned nil for a just-inserted row") - } - if got.ID != id { - t.Fatalf("round-tripped id = %d, want %d", got.ID, id) - } - if got.Name != b.Name || got.Workflow != b.Workflow || got.MappingID != b.MappingID { - t.Fatalf("round-tripped fields = %+v", got) - } - if got.Matcher.Source != b.Matcher.Source { - t.Fatalf("round-tripped matcher.source = %q, want %q", got.Matcher.Source, b.Matcher.Source) - } - if got.Status != binding.StatusDraft { - t.Fatalf("status = %q, want %q (insert forces draft regardless of caller)", got.Status, binding.StatusDraft) - } - if got.Version != 1 { - t.Fatalf("version = %d, want 1", got.Version) - } - if got.Secret != b.Secret { - t.Fatalf("secret round-trip mismatch") - } - if got.CreatedAt.IsZero() || got.UpdatedAt.IsZero() { - t.Fatalf("CreatedAt/UpdatedAt must be stamped, got %+v", got) - } -} - -// TestInsertBindingRoundTripsOwnerRepo covers the multi-repo fix: a -// binding pinned to a specific owner/repo persists and reads back -// unchanged, and UpdateBinding can change the pin the same way it -// changes any other editable field. -func TestInsertBindingRoundTripsOwnerRepo(t *testing.T) { - s := openTest(t) - b := testBinding("sentry") - b.Owner, b.Repo = "acme", "widget" - id, err := s.InsertBinding(t.Context(), b) - if err != nil { - t.Fatalf("InsertBinding: %v", err) - } - - got, err := s.GetBinding(t.Context(), id) - if err != nil || got == nil { - t.Fatalf("GetBinding: %+v, %v", got, err) - } - if got.Owner != "acme" || got.Repo != "widget" { - t.Fatalf("round-tripped owner/repo = %q/%q, want acme/widget", got.Owner, got.Repo) - } - - updated := *got - updated.Owner, updated.Repo = "other-org", "other-repo" - if err := s.UpdateBinding(t.Context(), updated); err != nil { - t.Fatalf("UpdateBinding: %v", err) - } - got, err = s.GetBinding(t.Context(), id) - if err != nil || got == nil { - t.Fatalf("GetBinding after update: %+v, %v", got, err) - } - if got.Owner != "other-org" || got.Repo != "other-repo" { - t.Fatalf("updated owner/repo = %q/%q, want other-org/other-repo", got.Owner, got.Repo) - } -} - -func TestOpenMigratesLegacyBindingsOwnerRepoColumns(t *testing.T) { - path := filepath.Join(t.TempDir(), "legacy.db") - raw, err := sql.Open("sqlite", path) - if err != nil { - t.Fatal(err) - } - _, err = raw.ExecContext(t.Context(), ` - CREATE TABLE bindings ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT NOT NULL DEFAULT '', - source TEXT NOT NULL DEFAULT '', - mapping_id INTEGER NOT NULL DEFAULT 0, - workflow TEXT NOT NULL DEFAULT '', - version INTEGER NOT NULL DEFAULT 1, - status TEXT NOT NULL DEFAULT 'draft', - secret TEXT NOT NULL DEFAULT '', - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL - ); - INSERT INTO bindings (name, source, workflow, secret, created_at, updated_at) - VALUES ('legacy', 'sentry', 'implement', 'legacy-secret', '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z'); - PRAGMA user_version = 2`) - if err != nil { - t.Fatal(err) - } - if err := raw.Close(); err != nil { - t.Fatal(err) - } - - s, err := Open(t.Context(), path) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = s.Close() }) - - got, err := s.ListBindings(t.Context()) - if err != nil { - t.Fatalf("ListBindings after legacy migration: %v", err) - } - if len(got) != 1 || got[0].Name != "legacy" || got[0].Owner != "" || got[0].Repo != "" { - t.Fatalf("legacy bindings = %+v, want preserved row with empty owner/repo", got) - } - - created := testBinding("github") - created.Owner, created.Repo = "acme", "widget" - id, err := s.InsertBinding(t.Context(), created) - if err != nil { - t.Fatalf("InsertBinding after legacy migration: %v", err) - } - created.ID = id - created.Secret = "" - created.Owner, created.Repo = "other", "repo" - if err := s.UpdateBinding(t.Context(), created); err != nil { - t.Fatalf("UpdateBinding after legacy migration: %v", err) - } - updated, err := s.GetBinding(t.Context(), id) - if err != nil || updated == nil { - t.Fatalf("GetBinding after legacy migration update: %+v, %v", updated, err) - } - if updated.Owner != "other" || updated.Repo != "repo" { - t.Fatalf("updated owner/repo = %q/%q, want other/repo", updated.Owner, updated.Repo) - } -} - -func TestGetBindingMissingReturnsNilNoError(t *testing.T) { - s := openTest(t) - got, err := s.GetBinding(t.Context(), 99999) - if err != nil { - t.Fatalf("GetBinding: %v", err) - } - if got != nil { - t.Fatalf("GetBinding for a missing id = %+v, want nil", got) - } -} - -func TestListBindingsOrdersNewestFirst(t *testing.T) { - s := openTest(t) - for i, src := range []string{"alpha", "beta", "gamma"} { - if _, err := s.InsertBinding(t.Context(), testBinding(src)); err != nil { - t.Fatalf("InsertBinding[%d]: %v", i, err) - } - } - got, err := s.ListBindings(t.Context()) - if err != nil { - t.Fatalf("ListBindings: %v", err) - } - if len(got) != 3 { - t.Fatalf("ListBindings = %d rows, want 3", len(got)) - } - if got[0].ID <= got[1].ID || got[1].ID <= got[2].ID { - t.Fatalf("ListBindings not newest-first: ids = %d, %d, %d", got[0].ID, got[1].ID, got[2].ID) - } -} - -func TestUpdateBindingBumpsVersion(t *testing.T) { - s := openTest(t) - id, err := s.InsertBinding(t.Context(), testBinding("sentry")) - if err != nil { - t.Fatalf("InsertBinding: %v", err) - } - updated := testBinding("sentry") - updated.ID = id - updated.Name = "renamed" - if err := s.UpdateBinding(t.Context(), updated); err != nil { - t.Fatalf("UpdateBinding: %v", err) - } - got, err := s.GetBinding(t.Context(), id) - if err != nil || got == nil { - t.Fatalf("GetBinding after update: %+v, %v", got, err) - } - if got.Version != 2 { - t.Fatalf("version = %d, want 2 after one update", got.Version) - } - if got.Name != "renamed" { - t.Fatalf("updated name = %q, want %q", got.Name, "renamed") - } -} - -func TestUpdateBindingOnArmedForcesPendingApproval(t *testing.T) { - s := openTest(t) - id, err := s.InsertBinding(t.Context(), testBinding("sentry")) - if err != nil { - t.Fatalf("InsertBinding: %v", err) - } - if _, err := s.db.ExecContext(t.Context(), `UPDATE bindings SET status='pending_approval' WHERE id=?`, id); err != nil { - t.Fatalf("force pending_approval: %v", err) - } - if err := s.ApproveBinding(t.Context(), id); err != nil { - t.Fatalf("ApproveBinding: %v", err) - } - armed, err := s.GetBinding(t.Context(), id) - if err != nil || armed == nil { - t.Fatalf("GetBinding after approve: %+v, %v", armed, err) - } - if armed.Status != binding.StatusArmed { - t.Fatalf("status after approve = %q, want armed", armed.Status) - } - - // An edit on an armed binding must drop it back to pending_approval. - updated := *armed - updated.Name = "edit while armed" - if err := s.UpdateBinding(t.Context(), updated); err != nil { - t.Fatalf("UpdateBinding on armed: %v", err) - } - got, err := s.GetBinding(t.Context(), id) - if err != nil || got == nil { - t.Fatalf("GetBinding after armed edit: %+v, %v", got, err) - } - if got.Status != binding.StatusPendingApproval { - t.Fatalf("status after armed edit = %q, want pending_approval", got.Status) - } - if got.Version <= armed.Version { - t.Fatalf("version did not advance after update: was %d, now %d", armed.Version, got.Version) - } -} - -func TestUpdateBindingPreservesSecretWhenEmpty(t *testing.T) { - s := openTest(t) - b := testBinding("sentry") - b.Secret = "0123456789abcdef0123456789abcdef" - id, err := s.InsertBinding(t.Context(), b) - if err != nil { - t.Fatalf("InsertBinding: %v", err) - } - updated := testBinding("sentry") - updated.ID = id - updated.Secret = "" // empty -> preserve - if err := s.UpdateBinding(t.Context(), updated); err != nil { - t.Fatalf("UpdateBinding: %v", err) - } - got, err := s.GetBinding(t.Context(), id) - if err != nil || got == nil { - t.Fatalf("GetBinding after update: %+v, %v", got, err) - } - if got.Secret != b.Secret { - t.Fatalf("secret = %q, want preserved %q", got.Secret, b.Secret) - } -} - -// TestUpdateBindingSecretWithCipher covers the same empty-secret-means-preserve -// contract as TestUpdateBindingPreservesSecretWhenEmpty, but with an at-rest -// cipher installed -- the configuration every deployment with -// [bindings].encryption_key actually runs. -// -// The sibling test opens the store without a cipher, where -// COALESCE(NULLIF(?, ”), secret) sees a literal empty string and holds. With a -// cipher the secret is encrypted BEFORE that guard runs, so an empty caller -// secret was sealed into a non-empty envelope, NULLIF stopped firing, and a -// name-only edit overwrote the binding's HMAC secret with an encrypted empty -// value -- after which the binding can never authenticate an inbound webhook. -// -// The second case is the other half of the contract: a genuinely supplied -// secret must still replace the stored one, so the fix cannot degrade into -// "updates never change the secret". -func TestUpdateBindingSecretWithCipher(t *testing.T) { - t.Parallel() - - const original = "0123456789abcdef0123456789abcdef" - replacement := strings.Repeat("f", len(original)) - - cases := []struct { - name string - newSecret string - want string - }{ - {"empty caller secret preserves the stored secret", "", original}, - {"a supplied secret replaces the stored secret", replacement, replacement}, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - - s := openTestWithBindingCipher(t) - b := testBinding("sentry") - b.Secret = original - id, err := s.InsertBinding(t.Context(), b) - if err != nil { - t.Fatalf("InsertBinding: %v", err) - } - - updated := testBinding("sentry") - updated.ID = id - updated.Name = "renamed binding" // an ordinary edit that does not touch the secret - updated.Secret = tc.newSecret - if err := s.UpdateBinding(t.Context(), updated); err != nil { - t.Fatalf("UpdateBinding: %v", err) - } - - got, err := s.GetBinding(t.Context(), id) - if err != nil || got == nil { - t.Fatalf("GetBinding after update: %+v, %v", got, err) - } - if got.Secret != tc.want { - t.Fatalf("secret = %q, want %q", got.Secret, tc.want) - } - }) - } -} - -func TestApproveBindingFromPendingApprovalToArmedSucceeds(t *testing.T) { - s := openTest(t) - id, err := s.InsertBinding(t.Context(), testBinding("sentry")) - if err != nil { - t.Fatalf("InsertBinding: %v", err) - } - if _, err := s.db.ExecContext(t.Context(), `UPDATE bindings SET status='pending_approval' WHERE id=?`, id); err != nil { - t.Fatalf("force pending_approval: %v", err) - } - if err := s.ApproveBinding(t.Context(), id); err != nil { - t.Fatalf("ApproveBinding: %v", err) - } - got, err := s.GetBinding(t.Context(), id) - if err != nil || got == nil { - t.Fatalf("GetBinding after approve: %+v, %v", got, err) - } - if got.Status != binding.StatusArmed { - t.Fatalf("status = %q, want armed", got.Status) - } -} - -func TestApproveBindingFromDraftFails(t *testing.T) { - s := openTest(t) - id, err := s.InsertBinding(t.Context(), testBinding("sentry")) - if err != nil { - t.Fatalf("InsertBinding: %v", err) - } - err = s.ApproveBinding(t.Context(), id) - if !errors.Is(err, ErrBindingTransition) { - t.Fatalf("ApproveBinding from draft = %v, want ErrBindingTransition", err) - } - got, err := s.GetBinding(t.Context(), id) - if err != nil || got == nil { - t.Fatalf("GetBinding after failed approve: %+v, %v", got, err) - } - if got.Status != binding.StatusDraft { - t.Fatalf("status after failed approve = %q, want unchanged draft", got.Status) - } -} - -func TestApproveBindingFromArmedFails(t *testing.T) { - s := openTest(t) - id, err := s.InsertBinding(t.Context(), testBinding("sentry")) - if err != nil { - t.Fatalf("InsertBinding: %v", err) - } - if _, err := s.db.ExecContext(t.Context(), `UPDATE bindings SET status='armed' WHERE id=?`, id); err != nil { - t.Fatalf("force armed: %v", err) - } - err = s.ApproveBinding(t.Context(), id) - if !errors.Is(err, ErrBindingTransition) { - t.Fatalf("ApproveBinding from armed = %v, want ErrBindingTransition", err) - } -} - -func TestApproveBindingMissingReturnsErrBindingNotFound(t *testing.T) { - s := openTest(t) - err := s.ApproveBinding(t.Context(), 99999) - if !errors.Is(err, ErrBindingNotFound) { - t.Fatalf("ApproveBinding on missing id = %v, want ErrBindingNotFound", err) - } -} - -func TestInsertBindingRejectsOverlap(t *testing.T) { - s := openTest(t) - if _, err := s.InsertBinding(t.Context(), testBinding("sentry")); err != nil { - t.Fatalf("first InsertBinding: %v", err) - } - _, err := s.InsertBinding(t.Context(), testBinding("sentry")) - if !errors.Is(err, ErrBindingOverlap) { - t.Fatalf("second InsertBinding (same source) = %v, want ErrBindingOverlap", err) - } -} - -func TestApproveBindingRejectsOverlapAfterSecondInserted(t *testing.T) { - s := openTest(t) - first, err := s.InsertBinding(t.Context(), testBinding("sentry")) - if err != nil { - t.Fatalf("first InsertBinding: %v", err) - } - if _, err := s.db.ExecContext(t.Context(), `UPDATE bindings SET status='pending_approval' WHERE id=?`, first); err != nil { - t.Fatalf("force pending_approval: %v", err) - } - if err := s.ApproveBinding(t.Context(), first); err != nil { - t.Fatalf("ApproveBinding first: %v", err) - } - - second, err := s.InsertBinding(t.Context(), testBinding("sentry")) - if err == nil { - // A duplicate-source insert must be rejected outright, so reaching - // here means the overlap guard failed at insert time. - t.Fatalf("second InsertBinding for armed source = id %d, want ErrBindingOverlap", second) - } - if !errors.Is(err, ErrBindingOverlap) { - t.Fatalf("second InsertBinding = %v, want ErrBindingOverlap", err) - } -} - -func TestDeleteBindingRemovesRowAndLedgerEntries(t *testing.T) { - s := openTest(t) - id, err := s.InsertBinding(t.Context(), testBinding("sentry")) - if err != nil { - t.Fatalf("InsertBinding: %v", err) - } - if err := s.RecordDispatch(t.Context(), id, 1, 42, 7); err != nil { - t.Fatalf("RecordDispatch: %v", err) - } - if err := s.DeleteBinding(t.Context(), id); err != nil { - t.Fatalf("DeleteBinding: %v", err) - } - got, err := s.GetBinding(t.Context(), id) - if err != nil { - t.Fatalf("GetBinding after delete: %v", err) - } - if got != nil { - t.Fatalf("GetBinding after delete = %+v, want nil", got) - } - var n int - if err := s.db.QueryRowContext(t.Context(), `SELECT COUNT(*) FROM binding_dispatches WHERE binding_id=?`, id).Scan(&n); err != nil { - t.Fatalf("COUNT binding_dispatches: %v", err) - } - if n != 0 { - t.Fatalf("binding_dispatches rows for binding_id=%d = %d, want 0", id, n) - } -} - -func TestRecordDispatchIsIdempotent(t *testing.T) { - s := openTest(t) - id, err := s.InsertBinding(t.Context(), testBinding("sentry")) - if err != nil { - t.Fatalf("InsertBinding: %v", err) - } - ctx := t.Context() - if err := s.RecordDispatch(ctx, id, 1, 42, 7); err != nil { - t.Fatalf("first RecordDispatch: %v", err) - } - - err = s.RecordDispatch(ctx, id, 1, 42, 99) - if !errors.Is(err, ErrAlreadyDispatched) { - t.Fatalf("second RecordDispatch (same binding/capture) = %v, want ErrAlreadyDispatched", err) - } -} - -func TestListUndispatchedCapturesExcludesDispatched(t *testing.T) { - s := openTest(t) - id, err := s.InsertBinding(t.Context(), testBinding("sentry")) - if err != nil { - t.Fatalf("InsertBinding: %v", err) - } - if _, err := s.db.ExecContext(t.Context(), `UPDATE bindings SET status='armed' WHERE id=?`, id); err != nil { - t.Fatalf("arm binding: %v", err) - } - - // Capture 1: authenticated, will be dispatched -> should NOT appear. - cap1, err := s.InsertCapture(t.Context(), CapturedEvent{ - Source: "sentry", Authenticated: true, Body: `{"id":"1"}`, - }, 0, 0) - if err != nil { - t.Fatalf("InsertCapture 1: %v", err) - } - // Capture 2: authenticated, not dispatched -> SHOULD appear. - cap2, err := s.InsertCapture(t.Context(), CapturedEvent{ - Source: "sentry", Authenticated: true, Body: `{"id":"2"}`, - }, 0, 0) - if err != nil { - t.Fatalf("InsertCapture 2: %v", err) - } - // Capture 3: unauthenticated -> should NOT appear (matches() requires auth). - cap3, err := s.InsertCapture(t.Context(), CapturedEvent{ - Source: "sentry", Authenticated: false, Body: `{"id":"3"}`, - }, 0, 0) - if err != nil { - t.Fatalf("InsertCapture 3: %v", err) - } - - ctx := t.Context() - if err := s.RecordDispatch(ctx, id, 1, cap1, 100); err != nil { - t.Fatalf("RecordDispatch: %v", err) - } - - got, err := s.ListUndispatchedCaptures(ctx, []string{"sentry"}, 10) - if err != nil { - t.Fatalf("ListUndispatchedCaptures: %v", err) - } - if len(got) != 1 { - t.Fatalf("ListUndispatchedCaptures = %d rows, want 1", len(got)) - } - if got[0].ID != cap2 { - t.Fatalf("returned capture id = %d, want %d (cap3 unauthenticated, cap1 dispatched)", got[0].ID, cap2) - } - _ = cap3 -} - -func TestListUndispatchedCapturesEmptySourcesReturnsNil(t *testing.T) { - s := openTest(t) - got, err := s.ListUndispatchedCaptures(t.Context(), nil, 10) - if err != nil { - t.Fatalf("ListUndispatchedCaptures(nil sources): %v", err) - } - if got != nil { - t.Fatalf("ListUndispatchedCaptures(nil sources) = %+v, want nil", got) - } -} diff --git a/internal/store/captures.go b/internal/store/captures.go deleted file mode 100644 index 8834d8938..000000000 --- a/internal/store/captures.go +++ /dev/null @@ -1,124 +0,0 @@ -package store - -import ( - "context" - "errors" - "fmt" - "time" -) - -const capturesSchema = ` -CREATE TABLE IF NOT EXISTS captured_events ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - received_at TEXT NOT NULL, - source TEXT NOT NULL DEFAULT '', - remote_addr TEXT NOT NULL DEFAULT '', - content_type TEXT NOT NULL DEFAULT '', - headers TEXT NOT NULL DEFAULT '{}', - body TEXT NOT NULL DEFAULT '', - authenticated INTEGER NOT NULL DEFAULT 0 -); -CREATE INDEX IF NOT EXISTS idx_captured_events_source ON captured_events(source, id); -` - -// captureTimeLayout is fixed-width (unlike time.RFC3339Nano, whose -// fractional-seconds field is trimmed of trailing zeros and omitted -// entirely when zero). A variable-width fractional part breaks lexicographic -// string comparison -- "15:04:05Z" (an exact second) sorts AFTER -// "15:04:05.1Z" (0.1s later) because 'Z' > '.', which would make the -// retention prune's `WHERE received_at < ?` unreliable. Second precision is -// enough here: row order already comes from id (autoincrement), and -// retention windows are measured in days, not sub-second intervals. -const captureTimeLayout = time.RFC3339 - -// InsertCapture persists a captured event and prunes the table in the same -// transaction: rows older than retention (when > 0) and rows beyond the -// newest maxEvents (when > 0) are deleted. Either bound of 0 or less -// disables that prune. See docs/prds/event-capture-storage.md's disk-bound -// mechanics -- this is one of three independent bounds, the other two -// (per-request size cap, per-source rate limit) are enforced by the caller -// before the body ever reaches this method. -func (s *Store) InsertCapture(ctx context.Context, c CapturedEvent, retention time.Duration, maxEvents int) (int64, error) { - tx, err := s.db.BeginTx(ctx, nil) - if err != nil { - return 0, err - } - defer func() { _ = tx.Rollback() }() - - if c.ReceivedAt.IsZero() { - c.ReceivedAt = time.Now().UTC() - } - res, err := tx.ExecContext(ctx, ` - INSERT INTO captured_events (received_at, source, remote_addr, content_type, headers, body, authenticated) - VALUES (?, ?, ?, ?, ?, ?, ?)`, - c.ReceivedAt.UTC().Format(captureTimeLayout), c.Source, c.RemoteAddr, - c.ContentType, c.Headers, c.Body, c.Authenticated) - if err != nil { - return 0, err - } - id, err := res.LastInsertId() - if err != nil { - return 0, err - } - - if retention > 0 { - cutoff := time.Now().UTC().Add(-retention).Format(captureTimeLayout) - if _, err := tx.ExecContext(ctx, `DELETE FROM captured_events WHERE received_at < ?`, cutoff); err != nil { - return 0, err - } - } - if maxEvents > 0 { - if _, err := tx.ExecContext(ctx, ` - DELETE FROM captured_events WHERE id NOT IN ( - SELECT id FROM captured_events ORDER BY id DESC LIMIT ? - )`, maxEvents); err != nil { - return 0, err - } - } - - if err := tx.Commit(); err != nil { - return 0, err - } - return id, nil -} - -// ListCaptures returns up to limit captured events, newest first. -func (s *Store) ListCaptures(ctx context.Context, limit int) (out []CapturedEvent, retErr error) { - rows, err := s.db.QueryContext(ctx, ` - SELECT id, received_at, source, remote_addr, content_type, headers, body, authenticated - FROM captured_events ORDER BY id DESC LIMIT ?`, limit) - if err != nil { - return nil, err - } - defer func() { retErr = errors.Join(retErr, rows.Close()) }() - - for rows.Next() { - var c CapturedEvent - var receivedAt string - var authenticated int - if err := rows.Scan(&c.ID, &receivedAt, &c.Source, &c.RemoteAddr, - &c.ContentType, &c.Headers, &c.Body, &authenticated); err != nil { - return nil, err - } - parsed, err := parseCaptureTime(receivedAt) - if err != nil { - return nil, err - } - c.ReceivedAt = parsed - c.Authenticated = authenticated != 0 - out = append(out, c) - } - return out, rows.Err() -} - -func parseCaptureTime(v string) (time.Time, error) { - if v == "" { - return time.Time{}, nil - } - for _, layout := range []string{time.RFC3339Nano, time.RFC3339} { - if parsed, err := time.Parse(layout, v); err == nil { - return parsed.UTC(), nil - } - } - return time.Time{}, fmt.Errorf("store: unrecognised capture timestamp %q", v) -} diff --git a/internal/store/captures_test.go b/internal/store/captures_test.go deleted file mode 100644 index 36c9eeda2..000000000 --- a/internal/store/captures_test.go +++ /dev/null @@ -1,169 +0,0 @@ -package store - -import ( - "testing" - "time" -) - -func TestInsertCaptureRoundTrips(t *testing.T) { - s := openTest(t) - id, err := s.InsertCapture(t.Context(), CapturedEvent{ - Source: "github", - RemoteAddr: "203.0.113.5", - ContentType: "application/json", - Headers: `{"x-hub-signature-256":"[redacted]"}`, - Body: `{"action":"opened"}`, - Authenticated: false, - }, 7*24*time.Hour, 0) - if err != nil { - t.Fatalf("InsertCapture: %v", err) - } - if id == 0 { - t.Fatalf("InsertCapture returned zero id") - } - - got, err := s.ListCaptures(t.Context(), 10) - if err != nil { - t.Fatalf("ListCaptures: %v", err) - } - if len(got) != 1 { - t.Fatalf("ListCaptures = %d rows, want 1", len(got)) - } - c := got[0] - if c.ID != id || c.Source != "github" || c.RemoteAddr != "203.0.113.5" || - c.ContentType != "application/json" || c.Body != `{"action":"opened"}` || - c.Authenticated { - t.Fatalf("round-tripped capture = %+v, want matching fields", c) - } - if c.ReceivedAt.IsZero() { - t.Fatalf("ReceivedAt is zero, want a timestamp stamped at insert") - } -} - -func TestListCapturesOrdersNewestFirst(t *testing.T) { - s := openTest(t) - for i := range 3 { - if _, err := s.InsertCapture(t.Context(), CapturedEvent{Source: "src", Body: "{}"}, 0, 0); err != nil { - t.Fatalf("InsertCapture[%d]: %v", i, err) - } - } - got, err := s.ListCaptures(t.Context(), 10) - if err != nil { - t.Fatalf("ListCaptures: %v", err) - } - if len(got) != 3 { - t.Fatalf("ListCaptures = %d rows, want 3", len(got)) - } - if got[0].ID <= got[1].ID || got[1].ID <= got[2].ID { - t.Fatalf("ListCaptures not newest-first: ids = %d, %d, %d", got[0].ID, got[1].ID, got[2].ID) - } -} - -func TestInsertCapturePrunesBeyondMaxEvents(t *testing.T) { - s := openTest(t) - var lastID int64 - for i := range 5 { - id, err := s.InsertCapture(t.Context(), CapturedEvent{Source: "src", Body: "{}"}, 0, 3) - if err != nil { - t.Fatalf("InsertCapture[%d]: %v", i, err) - } - lastID = id - } - got, err := s.ListCaptures(t.Context(), 100) - if err != nil { - t.Fatalf("ListCaptures: %v", err) - } - if len(got) != 3 { - t.Fatalf("row count = %d, want 3 (maxEvents cap)", len(got)) - } - if got[0].ID != lastID { - t.Fatalf("newest surviving row id = %d, want %d (the last insert)", got[0].ID, lastID) - } - // The two oldest of the five inserts must have been pruned. - for _, c := range got { - if c.ID <= lastID-3 { - t.Fatalf("found pruned row id %d still present after maxEvents=3 cap", c.ID) - } - } -} - -func TestInsertCapturePrunesOlderThanRetention(t *testing.T) { - s := openTest(t) - // Insert a row and manually age it beyond the retention window, since - // InsertCapture always stamps "now" and the test cannot wait real days. - id, err := s.InsertCapture(t.Context(), CapturedEvent{Source: "old", Body: "{}"}, 0, 0) - if err != nil { - t.Fatalf("InsertCapture: %v", err) - } - stale := time.Now().UTC().Add(-8 * 24 * time.Hour).Format(captureTimeLayout) - if _, err := s.db.ExecContext(t.Context(), - `UPDATE captured_events SET received_at = ? WHERE id = ?`, stale, id); err != nil { - t.Fatalf("backdate row: %v", err) - } - - // A fresh insert with a 7-day retention should prune the now-stale row. - if _, err := s.InsertCapture(t.Context(), CapturedEvent{Source: "new", Body: "{}"}, 7*24*time.Hour, 0); err != nil { - t.Fatalf("InsertCapture: %v", err) - } - - got, err := s.ListCaptures(t.Context(), 100) - if err != nil { - t.Fatalf("ListCaptures: %v", err) - } - for _, c := range got { - if c.ID == id { - t.Fatalf("stale row id %d survived a 7-day retention prune", id) - } - } - if len(got) != 1 { - t.Fatalf("row count = %d, want 1 (only the fresh insert)", len(got)) - } -} - -// TestCaptureTimeLayoutSortsLexicographicallyLikeChronologically pins a real -// defect that time.RFC3339Nano has: it trims trailing fractional-second -// zeros and omits the fractional part entirely when it is exactly zero, so -// "15:04:05Z" (an exact second) sorts LEXICOGRAPHICALLY AFTER "15:04:05.1Z" -// (0.1s later) because 'Z' (0x5A) > '.' (0x2E) -- backwards from -// chronological order. A retention prune using `WHERE received_at < ?` on a -// column written that way is unreliable within the same second. -// captureTimeLayout must be fixed-width so string order and time order -// always agree. -// -// This compares captureTimeLayout's string output directly rather than -// going through InsertCapture's real prune, because the defect only shows up -// when two timestamps share the same whole-second value and differ only in -// fractional presence -- a case InsertCapture's own cutoff (computed from -// the real wall clock) cannot be steered into deterministically in a test. -func TestCaptureTimeLayoutSortsLexicographicallyLikeChronologically(t *testing.T) { - exactSecond := time.Date(2026, 1, 1, 15, 4, 5, 0, time.UTC) - fractionLater := time.Date(2026, 1, 1, 15, 4, 5, 100_000_000, time.UTC) // same second, 0.1s later - - if !exactSecond.Before(fractionLater) { - t.Fatalf("test setup: %v must be before %v", exactSecond, fractionLater) - } - es, fs := exactSecond.Format(captureTimeLayout), fractionLater.Format(captureTimeLayout) - // The layout is second-precision, so two timestamps within the same - // second are allowed to format identically (that's the accepted - // precision tradeoff, not a bug) -- but a later time must never sort as - // lexicographically LESS than an earlier one. - if es > fs { - t.Fatalf("captureTimeLayout string order disagrees with chronological order: %q should not sort after %q", es, fs) - } -} - -func TestInsertCaptureZeroRetentionAndMaxEventsMeansUnbounded(t *testing.T) { - s := openTest(t) - for i := range 10 { - if _, err := s.InsertCapture(t.Context(), CapturedEvent{Source: "src", Body: "{}"}, 0, 0); err != nil { - t.Fatalf("InsertCapture[%d]: %v", i, err) - } - } - got, err := s.ListCaptures(t.Context(), 100) - if err != nil { - t.Fatalf("ListCaptures: %v", err) - } - if len(got) != 10 { - t.Fatalf("row count = %d, want 10 (retention=0, maxEvents=0 disables both prunes)", len(got)) - } -} diff --git a/internal/store/interface.go b/internal/store/interface.go index 3e3b7574a..3f0c618f0 100644 --- a/internal/store/interface.go +++ b/internal/store/interface.go @@ -61,21 +61,3 @@ var _ ChannelStatusStore = (*Store)(nil) // Compile-time check: *Store satisfies workflow.Store. var _ workflow.Store = (*Store)(nil) - -// Compile-time check: *Store satisfies CaptureStore. -var _ CaptureStore = (*Store)(nil) - -// Compile-time check: *Store satisfies MappingStore. -var _ MappingStore = (*Store)(nil) - -// Compile-time check: *Store satisfies BindingStore. -var _ BindingStore = (*Store)(nil) - -// Compile-time check: *Store satisfies BindingDispatcher. -var _ BindingDispatcher = (*Store)(nil) - -// Compile-time check: *Store satisfies BindingTaskCreator. -var _ BindingTaskCreator = (*Store)(nil) - -// Compile-time check: *Store satisfies PlaybookDispatcher. -var _ PlaybookDispatcher = (*Store)(nil) diff --git a/internal/store/mappings.go b/internal/store/mappings.go deleted file mode 100644 index 51e7221bd..000000000 --- a/internal/store/mappings.go +++ /dev/null @@ -1,162 +0,0 @@ -package store - -import ( - "context" - "database/sql" - "encoding/json" - "errors" - "fmt" - "time" - - "github.com/samcharles93/archie-core/internal/domain/mapping" -) - -const mappingsSchema = ` -CREATE TABLE IF NOT EXISTS field_mappings ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT NOT NULL DEFAULT '', - source_hint TEXT NOT NULL DEFAULT '', - fields_json TEXT NOT NULL DEFAULT '[]', - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL -); -` - -// ErrMappingNotFound is returned by UpdateMapping and DeleteMapping when no -// row matches the given ID. GetMapping uses (nil, nil) instead, matching -// TaskByID's convention -- "not found" is the normal answer to "does a -// mapping with this ID currently exist," not an error. - -// mappingTimeLayout follows captureTimeLayout's reasoning: fixed-width so -// string and chronological order agree, even though nothing here prunes on -// a string comparison today -- consistency with the rest of the store's -// timestamp columns. -const mappingTimeLayout = time.RFC3339 - -// InsertMapping persists a new mapping and stamps CreatedAt/UpdatedAt to -// now, ignoring any caller-supplied values. -func (s *Store) InsertMapping(ctx context.Context, m mapping.Mapping) (int64, error) { - fieldsJSON, err := json.Marshal(m.Fields) - if err != nil { - return 0, fmt.Errorf("store: marshal mapping fields: %w", err) - } - now := time.Now().UTC().Format(mappingTimeLayout) - res, err := s.db.ExecContext(ctx, ` - INSERT INTO field_mappings (name, source_hint, fields_json, created_at, updated_at) - VALUES (?, ?, ?, ?, ?)`, - m.Name, m.SourceHint, string(fieldsJSON), now, now) - if err != nil { - return 0, err - } - return res.LastInsertId() -} - -// GetMapping returns the mapping with the given ID, or (nil, nil) if none -// exists. -func (s *Store) GetMapping(ctx context.Context, id int64) (*mapping.Mapping, error) { - row := s.db.QueryRowContext(ctx, ` - SELECT id, name, source_hint, fields_json, created_at, updated_at - FROM field_mappings WHERE id=?`, id) - m, err := scanMapping(row) - if errors.Is(err, sql.ErrNoRows) { - return nil, nil - } - if err != nil { - return nil, err - } - return m, nil -} - -// ListMappings returns every mapping, newest first. -func (s *Store) ListMappings(ctx context.Context) (out []mapping.Mapping, retErr error) { - rows, err := s.db.QueryContext(ctx, ` - SELECT id, name, source_hint, fields_json, created_at, updated_at - FROM field_mappings ORDER BY id DESC`) - if err != nil { - return nil, err - } - defer func() { retErr = errors.Join(retErr, rows.Close()) }() - - for rows.Next() { - m, err := scanMappingRow(rows) - if err != nil { - return nil, err - } - out = append(out, *m) - } - return out, rows.Err() -} - -// UpdateMapping replaces the name, source hint and fields of an existing -// mapping and bumps UpdatedAt to now. Returns ErrMappingNotFound if m.ID -// does not exist. -func (s *Store) UpdateMapping(ctx context.Context, m mapping.Mapping) error { - fieldsJSON, err := json.Marshal(m.Fields) - if err != nil { - return fmt.Errorf("store: marshal mapping fields: %w", err) - } - res, err := s.db.ExecContext(ctx, ` - UPDATE field_mappings SET name=?, source_hint=?, fields_json=?, updated_at=? - WHERE id=?`, - m.Name, m.SourceHint, string(fieldsJSON), time.Now().UTC().Format(mappingTimeLayout), m.ID) - if err != nil { - return err - } - n, err := res.RowsAffected() - if err != nil { - return err - } - if n == 0 { - return ErrMappingNotFound - } - return nil -} - -// DeleteMapping removes a mapping by ID. Returns ErrMappingNotFound if it -// does not exist. -func (s *Store) DeleteMapping(ctx context.Context, id int64) error { - res, err := s.db.ExecContext(ctx, `DELETE FROM field_mappings WHERE id=?`, id) - if err != nil { - return err - } - n, err := res.RowsAffected() - if err != nil { - return err - } - if n == 0 { - return ErrMappingNotFound - } - return nil -} - -// mappingScanner is satisfied by both *sql.Row and *sql.Rows, so -// scanMapping's column list stays in exactly one place. -type mappingScanner interface { - Scan(dest ...any) error -} - -func scanMapping(row mappingScanner) (*mapping.Mapping, error) { - return scanMappingRow(row) -} - -func scanMappingRow(row mappingScanner) (*mapping.Mapping, error) { - var m mapping.Mapping - var fieldsJSON, createdAt, updatedAt string - if err := row.Scan(&m.ID, &m.Name, &m.SourceHint, &fieldsJSON, &createdAt, &updatedAt); err != nil { - return nil, err - } - if err := json.Unmarshal([]byte(fieldsJSON), &m.Fields); err != nil { - return nil, fmt.Errorf("store: unmarshal mapping fields: %w", err) - } - created, err := time.Parse(mappingTimeLayout, createdAt) - if err != nil { - return nil, fmt.Errorf("store: unrecognised mapping created_at %q: %w", createdAt, err) - } - updated, err := time.Parse(mappingTimeLayout, updatedAt) - if err != nil { - return nil, fmt.Errorf("store: unrecognised mapping updated_at %q: %w", updatedAt, err) - } - m.CreatedAt = created.UTC() - m.UpdatedAt = updated.UTC() - return &m, nil -} diff --git a/internal/store/mappings_test.go b/internal/store/mappings_test.go deleted file mode 100644 index da7486b36..000000000 --- a/internal/store/mappings_test.go +++ /dev/null @@ -1,145 +0,0 @@ -package store - -import ( - "errors" - "testing" - - "github.com/samcharles93/archie-core/internal/domain/mapping" -) - -func testFields() []mapping.Field { - return []mapping.Field{ - {Name: "title", Path: "issue.title", Type: mapping.TypeString, Required: true}, - {Name: "count", Path: "count", Type: mapping.TypeNumber}, - } -} - -func TestInsertMappingRoundTrips(t *testing.T) { - s := openTest(t) - id, err := s.InsertMapping(t.Context(), mapping.Mapping{ - Name: "sentry issue opened", - SourceHint: "sentry", - Fields: testFields(), - }) - if err != nil { - t.Fatalf("InsertMapping: %v", err) - } - if id == 0 { - t.Fatalf("InsertMapping returned zero id") - } - - got, err := s.GetMapping(t.Context(), id) - if err != nil { - t.Fatalf("GetMapping: %v", err) - } - if got == nil { - t.Fatalf("GetMapping returned nil for a just-inserted row") - } - if got.ID != id || got.Name != "sentry issue opened" || got.SourceHint != "sentry" { - t.Fatalf("round-tripped mapping = %+v", got) - } - if len(got.Fields) != 2 || got.Fields[0].Name != "title" || got.Fields[0].Path != "issue.title" || - got.Fields[0].Type != mapping.TypeString || !got.Fields[0].Required { - t.Fatalf("round-tripped fields = %+v", got.Fields) - } - if got.CreatedAt.IsZero() || got.UpdatedAt.IsZero() { - t.Fatalf("CreatedAt/UpdatedAt must be stamped, got %+v", got) - } -} - -func TestGetMappingMissingReturnsNilNoError(t *testing.T) { - s := openTest(t) - got, err := s.GetMapping(t.Context(), 999) - if err != nil { - t.Fatalf("GetMapping: %v", err) - } - if got != nil { - t.Fatalf("GetMapping for a missing id = %+v, want nil", got) - } -} - -func TestListMappingsOrdersNewestFirst(t *testing.T) { - s := openTest(t) - for i := range 3 { - if _, err := s.InsertMapping(t.Context(), mapping.Mapping{Name: "m", Fields: testFields()}); err != nil { - t.Fatalf("InsertMapping[%d]: %v", i, err) - } - } - got, err := s.ListMappings(t.Context()) - if err != nil { - t.Fatalf("ListMappings: %v", err) - } - if len(got) != 3 { - t.Fatalf("ListMappings = %d rows, want 3", len(got)) - } - if got[0].ID <= got[1].ID || got[1].ID <= got[2].ID { - t.Fatalf("ListMappings not newest-first: ids = %d, %d, %d", got[0].ID, got[1].ID, got[2].ID) - } -} - -func TestUpdateMappingChangesFieldsAndBumpsUpdatedAt(t *testing.T) { - s := openTest(t) - id, err := s.InsertMapping(t.Context(), mapping.Mapping{Name: "orig", Fields: testFields()}) - if err != nil { - t.Fatalf("InsertMapping: %v", err) - } - before, err := s.GetMapping(t.Context(), id) - if err != nil || before == nil { - t.Fatalf("GetMapping before update: %+v, %v", before, err) - } - - newFields := []mapping.Field{{Name: "renamed", Path: "a.b", Type: mapping.TypeBool}} - if err := s.UpdateMapping(t.Context(), mapping.Mapping{ - ID: id, Name: "renamed mapping", SourceHint: "github", Fields: newFields, - }); err != nil { - t.Fatalf("UpdateMapping: %v", err) - } - - after, err := s.GetMapping(t.Context(), id) - if err != nil || after == nil { - t.Fatalf("GetMapping after update: %+v, %v", after, err) - } - if after.Name != "renamed mapping" || after.SourceHint != "github" { - t.Fatalf("updated mapping = %+v", after) - } - if len(after.Fields) != 1 || after.Fields[0].Name != "renamed" { - t.Fatalf("updated fields = %+v", after.Fields) - } - if !after.UpdatedAt.After(before.UpdatedAt) && !after.UpdatedAt.Equal(before.UpdatedAt) { - t.Fatalf("UpdatedAt did not advance: before=%v after=%v", before.UpdatedAt, after.UpdatedAt) - } -} - -func TestUpdateMappingMissingReturnsErrMappingNotFound(t *testing.T) { - s := openTest(t) - err := s.UpdateMapping(t.Context(), mapping.Mapping{ID: 999, Name: "x", Fields: testFields()}) - if !errors.Is(err, ErrMappingNotFound) { - t.Fatalf("UpdateMapping on missing id = %v, want ErrMappingNotFound", err) - } -} - -func TestDeleteMappingRemovesRow(t *testing.T) { - s := openTest(t) - id, err := s.InsertMapping(t.Context(), mapping.Mapping{Name: "m", Fields: testFields()}) - if err != nil { - t.Fatalf("InsertMapping: %v", err) - } - if err := s.DeleteMapping(t.Context(), id); err != nil { - t.Fatalf("DeleteMapping: %v", err) - } - got, err := s.GetMapping(t.Context(), id) - if err != nil { - t.Fatalf("GetMapping after delete: %v", err) - } - if got != nil { - t.Fatalf("GetMapping after delete = %+v, want nil", got) - } -} - -func TestDeleteMappingMissingReturnsErrMappingNotFound(t *testing.T) { - s := openTest(t) - err := s.DeleteMapping(t.Context(), 999) - if !errors.Is(err, ErrMappingNotFound) { - t.Fatalf("DeleteMapping on missing id = %v, want ErrMappingNotFound", err) - } -} diff --git a/internal/store/playbook_dispatches.go b/internal/store/playbook_dispatches.go deleted file mode 100644 index 97da05e2f..000000000 --- a/internal/store/playbook_dispatches.go +++ /dev/null @@ -1,62 +0,0 @@ -package store - -import ( - "context" - "time" -) - -// playbookDispatchesSchema is the durable idempotency ledger for -// side-effecting playbook actions (docs/prds/eda-playbook-engine.md gap 2). -// It copies binding_dispatches' conventions exactly: a durable table whose -// PRIMARY KEY is the structural (playbook_id, playbook_version, event_id, -// action_id) tuple, written with INSERT OR IGNORE so a duplicate is a no-op -// write rather than a constraint error. -const playbookDispatchesSchema = ` -CREATE TABLE IF NOT EXISTS playbook_dispatches ( - playbook_id TEXT NOT NULL, - playbook_version TEXT NOT NULL, - event_id TEXT NOT NULL, - action_id TEXT NOT NULL, - dispatched_at TEXT NOT NULL, - PRIMARY KEY (playbook_id, playbook_version, event_id, action_id) -); -` - -// RecordPlaybookDispatch writes one (playbook, version, event, action) dedup -// row. INSERT OR IGNORE + RowsAffected is the dedup test: a duplicate is a -// no-op write and returns ErrAlreadyDispatched rather than a constraint -// error. The row is written and committed BEFORE the action's side effect is -// invoked by the coordinator, which is the deliberate reverse of -// binding_dispatches: an at-most-once ledger for non-revocable side effects. -func (s *Store) RecordPlaybookDispatch( - ctx context.Context, - playbookID string, - playbookVersion string, - eventID string, - actionID string, -) error { - res, err := s.db.ExecContext(ctx, ` - INSERT OR IGNORE INTO playbook_dispatches (playbook_id, playbook_version, event_id, action_id, dispatched_at) - VALUES (?, ?, ?, ?, ?)`, - playbookID, playbookVersion, eventID, actionID, - time.Now().UTC().Format(bindingTimeLayout)) - if err != nil { - return err - } - n, err := res.RowsAffected() - if err != nil { - return err - } - if n == 0 { - return ErrAlreadyDispatched - } - return nil -} - -// DeletePlaybookDispatches removes every ledger row for the given playbook, -// mirroring DeleteBinding's row cleanup so a recreated playbook does not -// inherit stale dedup state. -func (s *Store) DeletePlaybookDispatches(ctx context.Context, playbookID string) error { - _, err := s.db.ExecContext(ctx, `DELETE FROM playbook_dispatches WHERE playbook_id = ?`, playbookID) - return err -} diff --git a/internal/store/playbook_dispatches_test.go b/internal/store/playbook_dispatches_test.go deleted file mode 100644 index 75b85bc41..000000000 --- a/internal/store/playbook_dispatches_test.go +++ /dev/null @@ -1,73 +0,0 @@ -package store - -import ( - "errors" - "testing" -) - -func TestRecordPlaybookDispatchIsIdempotent(t *testing.T) { - s := openTest(t) - ctx := t.Context() - if err := s.RecordPlaybookDispatch(ctx, "pb.yaml", "abc123", "archie:acme/widget/7", "notify"); err != nil { - t.Fatalf("first RecordPlaybookDispatch: %v", err) - } - err := s.RecordPlaybookDispatch(ctx, "pb.yaml", "abc123", "archie:acme/widget/7", "notify") - if !errors.Is(err, ErrAlreadyDispatched) { - t.Fatalf("second RecordPlaybookDispatch = %v, want ErrAlreadyDispatched", err) - } -} - -func TestRecordPlaybookDispatchKeyIsAllFourComponents(t *testing.T) { - s := openTest(t) - ctx := t.Context() - - if err := s.RecordPlaybookDispatch(ctx, "pb", "v1", "e1", "notify"); err != nil { - t.Fatalf("record (pb, v1, e1, notify): %v", err) - } - // Every component of the PRIMARY KEY distinguishes a row: a mutant key - // missing any one of the four would collapse distinct dispatches here. - if err := s.RecordPlaybookDispatch(ctx, "pb", "v1", "e1", "build"); err != nil { - t.Fatalf("record (pb, v1, e1, build) = %v, want success (action_id is a key component)", err) - } - if err := s.RecordPlaybookDispatch(ctx, "pb", "v1", "e2", "notify"); err != nil { - t.Fatalf("record (pb, v1, e2, notify) = %v, want success (event_id is a key component)", err) - } - if err := s.RecordPlaybookDispatch(ctx, "pb", "v2", "e1", "notify"); err != nil { - t.Fatalf("record (pb, v2, e1, notify) = %v, want success (playbook_version is a key component)", err) - } - if err := s.RecordPlaybookDispatch(ctx, "pb2", "v1", "e1", "notify"); err != nil { - t.Fatalf("record (pb2, v1, e1, notify) = %v, want success (playbook_id is a key component)", err) - } - if err := s.RecordPlaybookDispatch(ctx, "pb", "v1", "e1", "notify"); !errors.Is(err, ErrAlreadyDispatched) { - t.Fatalf("re-record (pb, v1, e1, notify) = %v, want ErrAlreadyDispatched", err) - } -} - -func TestDeletePlaybookDispatchesRemovesOnlyThatPlaybook(t *testing.T) { - s := openTest(t) - ctx := t.Context() - if err := s.RecordPlaybookDispatch(ctx, "pb.yaml", "v1", "archie:acme/widget/7", "notify"); err != nil { - t.Fatalf("RecordPlaybookDispatch pb.yaml: %v", err) - } - if err := s.RecordPlaybookDispatch(ctx, "other.yaml", "v2", "archie:acme/widget/8", "notify"); err != nil { - t.Fatalf("RecordPlaybookDispatch other.yaml: %v", err) - } - - if err := s.DeletePlaybookDispatches(ctx, "pb.yaml"); err != nil { - t.Fatalf("DeletePlaybookDispatches: %v", err) - } - - var deleted, other int - if err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM playbook_dispatches WHERE playbook_id=?`, "pb.yaml").Scan(&deleted); err != nil { - t.Fatalf("COUNT pb.yaml: %v", err) - } - if err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM playbook_dispatches WHERE playbook_id=?`, "other.yaml").Scan(&other); err != nil { - t.Fatalf("COUNT other.yaml: %v", err) - } - if deleted != 0 { - t.Fatalf("playbook_dispatches rows for pb.yaml = %d, want 0", deleted) - } - if other != 1 { - t.Fatalf("playbook_dispatches rows for other.yaml = %d, want 1 (delete must not touch other playbooks)", other) - } -} diff --git a/internal/store/store.go b/internal/store/store.go index 09e92d550..066ead888 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -20,8 +20,7 @@ import ( ) type Store struct { - db *sql.DB - bindingsCipher BindingCipher + db *sql.DB } // taskSchemaVersion is the newest schema this binary opens. ValidateFile @@ -32,16 +31,7 @@ const taskSchemaVersion = 4 // OpenOption configures the store at open time. type OpenOption func(*openOptions) -type openOptions struct { - bindingsCipher BindingCipher -} - -// WithBindingCipher installs an at-rest encryption cipher for binding secrets -// (docs/prds/binding-secret-encryption.md). When omitted, binding secrets are -// persisted as plaintext -- legacy behaviour, unchanged. -func WithBindingCipher(c BindingCipher) OpenOption { - return func(o *openOptions) { o.bindingsCipher = c } -} +type openOptions struct{} // Open opens (creating if needed) the SQLite database and its schema. func Open(ctx context.Context, path string, opts ...OpenOption) (*Store, error) { @@ -57,13 +47,13 @@ func Open(ctx context.Context, path string, opts ...OpenOption) (*Store, error) return nil, err } db.SetMaxOpenConns(1) - if _, err := db.ExecContext(ctx, schema+eventsSchema+capturesSchema+mappingsSchema+bindingsSchema+playbookDispatchesSchema+configSnapshotSchema+channelStatusSchema+applyStatusSchema+resourcesSchema+identitiesSchema); err != nil { + if _, err := db.ExecContext(ctx, schema+eventsSchema+configSnapshotSchema+channelStatusSchema+applyStatusSchema+resourcesSchema+identitiesSchema); err != nil { return nil, errors.Join(fmt.Errorf("store: init schema: %w", err), db.Close()) } if err := migrateTasks(ctx, db); err != nil { return nil, errors.Join(fmt.Errorf("store: migrate: %w", err), db.Close()) } - return &Store{db: db, bindingsCipher: o.bindingsCipher}, nil + return &Store{db: db}, nil } func sqliteDSN(path string) string { @@ -94,10 +84,6 @@ func migrateTasks(ctx context.Context, db *sql.DB) error { if err != nil { return err } - bindingColumns, err := tableColumns(ctx, tx, "bindings") - if err != nil { - return err - } // events is created by `CREATE TABLE IF NOT EXISTS` (eventsSchema), which // is a no-op against a database that already has the table. On an existing // archie.db this read plus the `events` arm below is therefore the ONLY @@ -117,25 +103,22 @@ func migrateTasks(ctx context.Context, db *sql.DB) error { {"tasks", "retry_count", `ALTER TABLE tasks ADD COLUMN retry_count INTEGER NOT NULL DEFAULT 0`}, {"tasks", "source", `ALTER TABLE tasks ADD COLUMN source TEXT NOT NULL DEFAULT 'forge'`}, {"tasks", "identity", `ALTER TABLE tasks ADD COLUMN identity TEXT NOT NULL DEFAULT ''`}, - {"tasks", "binding_id", `ALTER TABLE tasks ADD COLUMN binding_id INTEGER NOT NULL DEFAULT 0`}, + {"tasks", "binding_id", `ALTER TABLE tasks ADD COLUMN binding_id TEXT NOT NULL DEFAULT ''`}, {"tasks", "binding_version", `ALTER TABLE tasks ADD COLUMN binding_version INTEGER NOT NULL DEFAULT 0`}, {"tasks", "review_payload", `ALTER TABLE tasks ADD COLUMN review_payload TEXT NOT NULL DEFAULT ''`}, {"tasks", "workflow_definition_version", `ALTER TABLE tasks ADD COLUMN workflow_definition_version INTEGER NOT NULL DEFAULT 0`}, {"tasks", "workflow_definition_digest", `ALTER TABLE tasks ADD COLUMN workflow_definition_digest TEXT NOT NULL DEFAULT ''`}, {"tasks", "workflow_definition_yaml", `ALTER TABLE tasks ADD COLUMN workflow_definition_yaml TEXT NOT NULL DEFAULT ''`}, - {"bindings", "owner", `ALTER TABLE bindings ADD COLUMN owner TEXT NOT NULL DEFAULT ''`}, - {"bindings", "repo", `ALTER TABLE bindings ADD COLUMN repo TEXT NOT NULL DEFAULT ''`}, {"events", "attempt", `ALTER TABLE events ADD COLUMN attempt INTEGER NOT NULL DEFAULT 0`}, {"events", "actor_id", `ALTER TABLE events ADD COLUMN actor_id TEXT NOT NULL DEFAULT ''`}, {"events", "actor_kind", `ALTER TABLE events ADD COLUMN actor_kind TEXT NOT NULL DEFAULT ''`}, {"events", "principal_id", `ALTER TABLE events ADD COLUMN principal_id TEXT NOT NULL DEFAULT ''`}, } for _, migration := range migrations { + // Only tasks and events remain here; the event-capture tables moved + // to internal/infrastructure/edastore, which owns their schema. present := columns - switch migration.table { - case "bindings": - present = bindingColumns - case "events": + if migration.table == "events" { present = eventsColumns } if present[migration.column] { @@ -248,7 +231,11 @@ CREATE TABLE IF NOT EXISTS tasks ( retry_count INTEGER NOT NULL DEFAULT 0, source TEXT NOT NULL DEFAULT 'forge', identity TEXT NOT NULL DEFAULT '', - binding_id INTEGER NOT NULL DEFAULT 0, + -- binding_id carries the id of the edastore binding that produced this + -- task, or "" for a task nobody bound. It is TEXT because bindings are + -- PocketBase records; a database created before that stores the same + -- values in an INTEGER-affinity column, which SQLite keeps as TEXT. + binding_id TEXT NOT NULL DEFAULT '', binding_version INTEGER NOT NULL DEFAULT 0, review_payload TEXT NOT NULL DEFAULT '', workflow_definition_version INTEGER NOT NULL DEFAULT 0, @@ -344,7 +331,7 @@ func (s *Store) EnqueueChatTask(ctx context.Context, owner, repo, title, body, w // a future repair pass could backfill. This is the same // best-effort-provenance pattern the rest of the task lifecycle uses // for fields added after the row's primary insert. -func (s *Store) EnqueueBindingTask(ctx context.Context, owner, repo, title, body, wf, identity string, bindingID int64, bindingVersion int) (*workflow.Task, error) { +func (s *Store) EnqueueBindingTask(ctx context.Context, owner, repo, title, body, wf, identity, bindingID string, bindingVersion int) (*workflow.Task, error) { t, err := s.EnqueueChatTask(ctx, owner, repo, title, body, wf, identity) if err != nil { return nil, err diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 59de7c265..5d1a24eeb 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -486,7 +486,7 @@ func TestTasksBindingColumnsRoundTrip(t *testing.T) { } if _, err := s.db.ExecContext(ctx, `UPDATE tasks SET binding_id=?, binding_version=? WHERE id=?`, - int64(42), 3, task.ID); err != nil { + "rbind0000000001", 3, task.ID); err != nil { t.Fatalf("UPDATE binding provenance: %v", err) } @@ -501,8 +501,8 @@ func TestTasksBindingColumnsRoundTrip(t *testing.T) { if got == nil { t.Fatalf("%s: nil task", name) } - if got.BindingID != 42 || got.BindingVersion != 3 { - t.Fatalf("%s: BindingID=%d BindingVersion=%d, want 42/3", + if got.BindingID != "rbind0000000001" || got.BindingVersion != 3 { + t.Fatalf("%s: BindingID=%q BindingVersion=%d, want rbind0000000001/3", name, got.BindingID, got.BindingVersion) } } @@ -511,7 +511,7 @@ func TestTasksBindingColumnsRoundTrip(t *testing.T) { if err != nil { t.Fatalf("Tasks: %v", err) } - if len(listed) == 0 || listed[0].BindingID != 42 || listed[0].BindingVersion != 3 { + if len(listed) == 0 || listed[0].BindingID != "rbind0000000001" || listed[0].BindingVersion != 3 { t.Fatalf("Tasks() round-trip: %+v", listed) } } diff --git a/internal/webui/api_binding.go b/internal/webui/api_binding.go index 5490c9658..85d48cb63 100644 --- a/internal/webui/api_binding.go +++ b/internal/webui/api_binding.go @@ -4,7 +4,6 @@ import ( "encoding/json" "errors" "net/http" - "strconv" "github.com/samcharles93/archie-core/internal/domain/binding" "github.com/samcharles93/archie-core/internal/domain/storecontract" @@ -16,7 +15,7 @@ import ( type bindingRequest struct { Name string `json:"name"` Matcher binding.Matcher `json:"matcher"` - MappingID int64 `json:"mapping_id"` + MappingID string `json:"mapping_id"` Workflow string `json:"workflow"` // Owner and Repo optionally pin the binding to one configured repo, // for multi-repo deployments. Both empty is valid (falls back to the @@ -104,8 +103,8 @@ func (s *Server) handleBindingGet(w http.ResponseWriter, r *http.Request) { http.Error(w, "bindings not configured", http.StatusServiceUnavailable) return } - id, err := strconv.ParseInt(r.PathValue("id"), 10, 64) - if err != nil { + id := r.PathValue("id") + if id == "" { http.Error(w, "invalid binding id", http.StatusBadRequest) return } @@ -131,8 +130,8 @@ func (s *Server) handleBindingUpdate(w http.ResponseWriter, r *http.Request) { http.Error(w, "bindings not configured", http.StatusServiceUnavailable) return } - id, err := strconv.ParseInt(r.PathValue("id"), 10, 64) - if err != nil { + id := r.PathValue("id") + if id == "" { http.Error(w, "invalid binding id", http.StatusBadRequest) return } @@ -194,8 +193,8 @@ func (s *Server) handleBindingDelete(w http.ResponseWriter, r *http.Request) { http.Error(w, "bindings not configured", http.StatusServiceUnavailable) return } - id, err := strconv.ParseInt(r.PathValue("id"), 10, 64) - if err != nil { + id := r.PathValue("id") + if id == "" { http.Error(w, "invalid binding id", http.StatusBadRequest) return } @@ -219,8 +218,8 @@ func (s *Server) handleBindingApprove(w http.ResponseWriter, r *http.Request) { http.Error(w, "bindings not configured", http.StatusServiceUnavailable) return } - id, err := strconv.ParseInt(r.PathValue("id"), 10, 64) - if err != nil { + id := r.PathValue("id") + if id == "" { http.Error(w, "invalid binding id", http.StatusBadRequest) return } diff --git a/internal/webui/api_binding_test.go b/internal/webui/api_binding_test.go index 85a9f520c..91ebe44a9 100644 --- a/internal/webui/api_binding_test.go +++ b/internal/webui/api_binding_test.go @@ -5,13 +5,13 @@ import ( "log/slog" "net/http" "path/filepath" - "strconv" "strings" "testing" "github.com/samcharles93/archie-core/internal/domain/binding" "github.com/samcharles93/archie-core/internal/domain/mapping" "github.com/samcharles93/archie-core/internal/domain/workflow" + "github.com/samcharles93/archie-core/internal/infrastructure/edastore" "github.com/samcharles93/archie-core/internal/store" ) @@ -25,19 +25,20 @@ func bindingTestServer(t *testing.T) *Server { t.Fatal(err) } t.Cleanup(func() { _ = s.Close() }) + eda := edastore.OpenTest(t) return &Server{ Store: s, Log: slog.New(slog.DiscardHandler), - Mappings: s, - Captures: s, - Bindings: s, + Mappings: eda, + Captures: eda, + Bindings: eda, ControlPlane: workflowControlPlane(t, workflow.WorkflowDefinitionCollection{Definitions: []workflow.WorkflowDefinitionEntry{{ID: "implement", YAML: "id: implement\nsteps:\n - type: implement.prepare\n"}}}), } } // seedMapping inserts one payload mapping and returns its row ID, so // binding tests have a valid non-zero mapping_id without hand-rolling SQL. -func seedMapping(t *testing.T, srv *Server, name string) int64 { +func seedMapping(t *testing.T, srv *Server, name string) string { t.Helper() id, err := srv.Mappings.InsertMapping(t.Context(), mapping.Mapping{ Name: name, @@ -54,7 +55,7 @@ func seedMapping(t *testing.T, srv *Server, name string) int64 { // binding.Validate and server-side workflow validation. The Secret is 32 // bytes (above the 16-byte floor) so length never accidentally trips // validation in a test that isn't about secret length. -func validBindingRequest(suffix, source string, mappingID int64) map[string]any { +func validBindingRequest(suffix, source, mappingID string) map[string]any { return map[string]any{ "name": "binding " + suffix, "matcher": map[string]any{"source": source}, @@ -76,7 +77,7 @@ func TestHandleBindingCreateAndGet(t *testing.T) { if err := json.Unmarshal(w.Body.Bytes(), &created); err != nil { t.Fatalf("unmarshal created: %v", err) } - if created.ID == 0 { + if created.ID == "" { t.Fatalf("created.ID = 0; body = %s", w.Body.String()) } if created.Status != binding.StatusDraft { @@ -86,7 +87,7 @@ func TestHandleBindingCreateAndGet(t *testing.T) { t.Fatalf("created.Secret = %q, want \"\" (response must strip secret)", created.Secret) } - w = doJSON(t, srv, http.MethodGet, "/api/bindings/"+strconv.FormatInt(created.ID, 10), nil) + w = doJSON(t, srv, http.MethodGet, "/api/bindings/"+created.ID, nil) if w.Code != http.StatusOK { t.Fatalf("get status = %d, want %d; body = %s", w.Code, http.StatusOK, w.Body.String()) } @@ -95,7 +96,7 @@ func TestHandleBindingCreateAndGet(t *testing.T) { t.Fatalf("unmarshal got: %v", err) } if got.ID != created.ID { - t.Fatalf("got.ID = %d, want %d", got.ID, created.ID) + t.Fatalf("got.ID = %q, want %q", got.ID, created.ID) } if got.Secret != "" { t.Fatalf("get.Secret = %q, want \"\"", got.Secret) @@ -128,7 +129,7 @@ func TestHandleBindingCreateAndUpdateRoundTripsOwnerRepo(t *testing.T) { updateReq := validBindingRequest("a", "sentry", mappingID) updateReq["owner"] = "other-org" updateReq["repo"] = "other-repo" - w = doJSON(t, srv, http.MethodPatch, "/api/bindings/"+strconv.FormatInt(created.ID, 10), updateReq) + w = doJSON(t, srv, http.MethodPatch, "/api/bindings/"+created.ID, updateReq) if w.Code != http.StatusOK { t.Fatalf("update status = %d, want %d; body = %s", w.Code, http.StatusOK, w.Body.String()) } @@ -256,7 +257,7 @@ func TestHandleBindingUpdatePreservesOtherFields(t *testing.T) { // pending_approval yet, an edit on a draft row correctly stays // draft. This test pins the editable-fields half of the contract. update := validBindingRequest("renamed", "sentry", mappingID) - w = doJSON(t, srv, http.MethodPatch, "/api/bindings/"+strconv.FormatInt(created.ID, 10), update) + w = doJSON(t, srv, http.MethodPatch, "/api/bindings/"+created.ID, update) if w.Code != http.StatusOK { t.Fatalf("patch status = %d, want %d; body = %s", w.Code, http.StatusOK, w.Body.String()) } @@ -268,7 +269,7 @@ func TestHandleBindingUpdatePreservesOtherFields(t *testing.T) { t.Fatalf("updated.Name = %q, want %q", updated.Name, "binding renamed") } if updated.MappingID != mappingID { - t.Fatalf("updated.MappingID = %d, want %d", updated.MappingID, mappingID) + t.Fatalf("updated.MappingID = %q, want %q", updated.MappingID, mappingID) } // Status MUST not regress from a state the caller never asked for. if updated.Status != binding.StatusDraft && updated.Status != binding.StatusPendingApproval { @@ -291,7 +292,7 @@ func TestHandleBindingUpdateWithEmptySecretPreservesExisting(t *testing.T) { update := validBindingRequest("renamed", "sentry", mappingID) update["secret"] = "" - w = doJSON(t, srv, http.MethodPatch, "/api/bindings/"+strconv.FormatInt(created.ID, 10), update) + w = doJSON(t, srv, http.MethodPatch, "/api/bindings/"+created.ID, update) if w.Code != http.StatusOK { t.Fatalf("patch status = %d, want %d (empty secret must mean 'keep existing', not fail validation); body = %s", w.Code, http.StatusOK, w.Body.String()) } @@ -315,7 +316,7 @@ func TestHandleBindingApproveFromDraftRejected(t *testing.T) { // draft is NOT a valid source state for approve; the binding must // first land in pending_approval (via PATCH). Approve from draft // returns ErrBindingTransition -> 409. - w = doJSON(t, srv, http.MethodPost, "/api/bindings/"+strconv.FormatInt(created.ID, 10)+"/approve", nil) + w = doJSON(t, srv, http.MethodPost, "/api/bindings/"+created.ID+"/approve", nil) if w.Code != http.StatusConflict { t.Fatalf("status = %d, want %d; body = %s", w.Code, http.StatusConflict, w.Body.String()) } @@ -335,7 +336,7 @@ func TestHandleBindingApproveHappyPath(t *testing.T) { t.Fatalf("created status = %q, want %q", created.Status, binding.StatusDraft) } - w = doJSON(t, srv, http.MethodPatch, "/api/bindings/"+strconv.FormatInt(created.ID, 10), validBindingRequest("renamed", "sentry", mappingID)) + w = doJSON(t, srv, http.MethodPatch, "/api/bindings/"+created.ID, validBindingRequest("renamed", "sentry", mappingID)) if w.Code != http.StatusOK { t.Fatalf("patch status = %d, want %d; body = %s", w.Code, http.StatusOK, w.Body.String()) } @@ -345,7 +346,7 @@ func TestHandleBindingApproveHappyPath(t *testing.T) { t.Fatalf("patched status = %q, want %q", patched.Status, binding.StatusPendingApproval) } - w = doJSON(t, srv, http.MethodPost, "/api/bindings/"+strconv.FormatInt(created.ID, 10)+"/approve", nil) + w = doJSON(t, srv, http.MethodPost, "/api/bindings/"+created.ID+"/approve", nil) if w.Code != http.StatusOK { t.Fatalf("approve status = %d, want %d; body = %s", w.Code, http.StatusOK, w.Body.String()) } @@ -367,11 +368,11 @@ func TestHandleBindingEditOnArmedDropsToPendingApproval(t *testing.T) { _ = json.Unmarshal(w.Body.Bytes(), &created) // Drive it to armed via the happy path. - _ = doJSON(t, srv, http.MethodPatch, "/api/bindings/"+strconv.FormatInt(created.ID, 10), validBindingRequest("a", "sentry", mappingID)) - _ = doJSON(t, srv, http.MethodPost, "/api/bindings/"+strconv.FormatInt(created.ID, 10)+"/approve", nil) + _ = doJSON(t, srv, http.MethodPatch, "/api/bindings/"+created.ID, validBindingRequest("a", "sentry", mappingID)) + _ = doJSON(t, srv, http.MethodPost, "/api/bindings/"+created.ID+"/approve", nil) // Now edit. Status must drop back to pending_approval. - w = doJSON(t, srv, http.MethodPatch, "/api/bindings/"+strconv.FormatInt(created.ID, 10), validBindingRequest("a-edited", "sentry", mappingID)) + w = doJSON(t, srv, http.MethodPatch, "/api/bindings/"+created.ID, validBindingRequest("a-edited", "sentry", mappingID)) if w.Code != http.StatusOK { t.Fatalf("patch status = %d, want %d; body = %s", w.Code, http.StatusOK, w.Body.String()) } @@ -397,11 +398,11 @@ func TestHandleBindingDeleteRemovesRow(t *testing.T) { var created binding.Binding _ = json.Unmarshal(w.Body.Bytes(), &created) - w = doJSON(t, srv, http.MethodDelete, "/api/bindings/"+strconv.FormatInt(created.ID, 10), nil) + w = doJSON(t, srv, http.MethodDelete, "/api/bindings/"+created.ID, nil) if w.Code != http.StatusNoContent { t.Fatalf("delete status = %d, want %d; body = %s", w.Code, http.StatusNoContent, w.Body.String()) } - w = doJSON(t, srv, http.MethodGet, "/api/bindings/"+strconv.FormatInt(created.ID, 10), nil) + w = doJSON(t, srv, http.MethodGet, "/api/bindings/"+created.ID, nil) if w.Code != http.StatusNotFound { t.Fatalf("get after delete status = %d, want %d", w.Code, http.StatusNotFound) } @@ -428,7 +429,7 @@ func TestHandleBindingMutationsRequireCSRFHeader(t *testing.T) { if err := json.Unmarshal(w.Body.Bytes(), &created); err != nil { t.Fatalf("unmarshal created binding: %v; body = %s", err, w.Body.String()) } - id := strconv.FormatInt(created.ID, 10) + id := created.ID tests := []struct { name, method, path string diff --git a/internal/webui/api_capture_test.go b/internal/webui/api_capture_test.go index 79f19dddc..d1fd68eb9 100644 --- a/internal/webui/api_capture_test.go +++ b/internal/webui/api_capture_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/samcharles93/archie-core/internal/infrastructure/edastore" "github.com/samcharles93/archie-core/internal/store" ) @@ -25,16 +26,18 @@ func captureTestServer(t *testing.T) *Server { } t.Cleanup(func() { _ = s.Close() }) return &Server{ - Store: s, - Log: slog.New(slog.DiscardHandler), - Captures: s, + Store: s, + Log: slog.New(slog.DiscardHandler), + // Captures come from the event-capture store; the task store no + // longer holds them. + Captures: edastore.OpenTest(t), } } // seedCapture inserts a captured row directly through the CaptureStore. // These tests cover the list read; the write path itself lives in // internal/infrastructure/captureintake and is tested there. -func seedCapture(t *testing.T, srv *Server, source string) int64 { +func seedCapture(t *testing.T, srv *Server, source string) string { t.Helper() id, err := srv.Captures.InsertCapture(t.Context(), store.CapturedEvent{ ReceivedAt: time.Now().UTC(), diff --git a/internal/webui/api_mapping.go b/internal/webui/api_mapping.go index 2b5f62ab4..1a0cd343a 100644 --- a/internal/webui/api_mapping.go +++ b/internal/webui/api_mapping.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "net/http" - "strconv" "github.com/samcharles93/archie-core/internal/domain/mapping" "github.com/samcharles93/archie-core/internal/domain/storecontract" @@ -73,8 +72,8 @@ func (s *Server) handleMappingGet(w http.ResponseWriter, r *http.Request) { http.Error(w, "mappings not configured", http.StatusServiceUnavailable) return } - id, err := strconv.ParseInt(r.PathValue("id"), 10, 64) - if err != nil { + id := r.PathValue("id") + if id == "" { http.Error(w, "invalid mapping id", http.StatusBadRequest) return } @@ -99,8 +98,8 @@ func (s *Server) handleMappingUpdate(w http.ResponseWriter, r *http.Request) { http.Error(w, "mappings not configured", http.StatusServiceUnavailable) return } - id, err := strconv.ParseInt(r.PathValue("id"), 10, 64) - if err != nil { + id := r.PathValue("id") + if id == "" { http.Error(w, "invalid mapping id", http.StatusBadRequest) return } @@ -140,8 +139,8 @@ func (s *Server) handleMappingDelete(w http.ResponseWriter, r *http.Request) { http.Error(w, "mappings not configured", http.StatusServiceUnavailable) return } - id, err := strconv.ParseInt(r.PathValue("id"), 10, 64) - if err != nil { + id := r.PathValue("id") + if id == "" { http.Error(w, "invalid mapping id", http.StatusBadRequest) return } @@ -161,7 +160,7 @@ func (s *Server) handleMappingDelete(w http.ResponseWriter, r *http.Request) { // against a real captured event before saving anything -- Fields is loose // input, not a saved mapping ID, per docs/prds/payload-field-mapping.md. type mappingPreviewRequest struct { - CaptureID int64 `json:"capture_id"` + CaptureID string `json:"capture_id"` Fields []mapping.Field `json:"fields"` } @@ -201,7 +200,7 @@ func (s *Server) handleMappingPreview(w http.ResponseWriter, r *http.Request) { // exactly one reader, the dashboard list view, so a by-ID lookup was never // needed until preview), so this scans the newest window rather than // adding a new store method for a single low-volume caller. -func (s *Server) captureByID(ctx context.Context, id int64) (*storecontract.CapturedEvent, error) { +func (s *Server) captureByID(ctx context.Context, id string) (*storecontract.CapturedEvent, error) { captures, err := s.Captures.ListCaptures(ctx, mappingCaptureScanWindow(s.CaptureMaxEvents)) if err != nil { return nil, err diff --git a/internal/webui/api_mapping_test.go b/internal/webui/api_mapping_test.go index bc0896e65..d5306b9a0 100644 --- a/internal/webui/api_mapping_test.go +++ b/internal/webui/api_mapping_test.go @@ -7,10 +7,10 @@ import ( "net/http" "net/http/httptest" "path/filepath" - "strconv" "testing" "github.com/samcharles93/archie-core/internal/domain/mapping" + "github.com/samcharles93/archie-core/internal/infrastructure/edastore" "github.com/samcharles93/archie-core/internal/store" ) @@ -24,11 +24,12 @@ func mappingTestServer(t *testing.T) *Server { t.Fatal(err) } t.Cleanup(func() { _ = s.Close() }) + eda := edastore.OpenTest(t) return &Server{ Store: s, Log: slog.New(slog.DiscardHandler), - Mappings: s, - Captures: s, + Mappings: eda, + Captures: eda, } } @@ -86,11 +87,11 @@ func TestHandleMappingCreateAndGet(t *testing.T) { if err := json.Unmarshal(w.Body.Bytes(), &created); err != nil { t.Fatalf("unmarshal created mapping: %v", err) } - if created.ID == 0 || created.Name != "sentry issue opened" { + if created.ID == "" || created.Name != "sentry issue opened" { t.Fatalf("created mapping = %+v", created) } - w = doJSON(t, srv, http.MethodGet, "/api/mappings/"+strconv.FormatInt(created.ID, 10), nil) + w = doJSON(t, srv, http.MethodGet, "/api/mappings/"+created.ID, nil) if w.Code != http.StatusOK { t.Fatalf("get status = %d, want %d; body = %s", w.Code, http.StatusOK, w.Body.String()) } @@ -151,7 +152,7 @@ func TestHandleMappingUpdate(t *testing.T) { var created mapping.Mapping _ = json.Unmarshal(w.Body.Bytes(), &created) - w = doJSON(t, srv, http.MethodPatch, "/api/mappings/"+strconv.FormatInt(created.ID, 10), map[string]any{ + w = doJSON(t, srv, http.MethodPatch, "/api/mappings/"+created.ID, map[string]any{ "name": "renamed", "fields": []mapping.Field{{Name: "b", Path: "b", Type: mapping.TypeBool}}, }) @@ -159,7 +160,7 @@ func TestHandleMappingUpdate(t *testing.T) { t.Fatalf("update status = %d, want %d; body = %s", w.Code, http.StatusOK, w.Body.String()) } - w = doJSON(t, srv, http.MethodGet, "/api/mappings/"+strconv.FormatInt(created.ID, 10), nil) + w = doJSON(t, srv, http.MethodGet, "/api/mappings/"+created.ID, nil) var got mapping.Mapping _ = json.Unmarshal(w.Body.Bytes(), &got) if got.Name != "renamed" || len(got.Fields) != 1 || got.Fields[0].Name != "b" { @@ -187,11 +188,11 @@ func TestHandleMappingDelete(t *testing.T) { var created mapping.Mapping _ = json.Unmarshal(w.Body.Bytes(), &created) - w = doJSON(t, srv, http.MethodDelete, "/api/mappings/"+strconv.FormatInt(created.ID, 10), nil) + w = doJSON(t, srv, http.MethodDelete, "/api/mappings/"+created.ID, nil) if w.Code != http.StatusNoContent { t.Fatalf("delete status = %d, want %d; body = %s", w.Code, http.StatusNoContent, w.Body.String()) } - w = doJSON(t, srv, http.MethodGet, "/api/mappings/"+strconv.FormatInt(created.ID, 10), nil) + w = doJSON(t, srv, http.MethodGet, "/api/mappings/"+created.ID, nil) if w.Code != http.StatusNotFound { t.Fatalf("get after delete status = %d, want %d", w.Code, http.StatusNotFound) } @@ -235,7 +236,7 @@ func TestHandleMappingPreviewResolvesAgainstARealCapture(t *testing.T) { func TestHandleMappingPreviewUnknownCaptureReturns404(t *testing.T) { srv := mappingTestServer(t) w := doJSON(t, srv, http.MethodPost, "/api/mappings/preview", map[string]any{ - "capture_id": 999, + "capture_id": "rabsent00000000", "fields": []mapping.Field{{Name: "a", Path: "a", Type: mapping.TypeString}}, }) if w.Code != http.StatusNotFound { @@ -284,7 +285,7 @@ func TestHandleMappingMutationsRequireCSRFHeader(t *testing.T) { if err := json.Unmarshal(w.Body.Bytes(), &created); err != nil { t.Fatalf("unmarshal created mapping: %v; body = %s", err, w.Body.String()) } - id := strconv.FormatInt(created.ID, 10) + id := created.ID tests := []struct { name, method, path string diff --git a/internal/webui/remote_store_test.go b/internal/webui/remote_store_test.go index b7335dd88..445fbd101 100644 --- a/internal/webui/remote_store_test.go +++ b/internal/webui/remote_store_test.go @@ -21,6 +21,7 @@ import ( "google.golang.org/grpc" "github.com/samcharles93/archie-core/internal/domain/workflow" + "github.com/samcharles93/archie-core/internal/infrastructure/edastore" "github.com/samcharles93/archie-core/internal/infrastructure/staterpc" "github.com/samcharles93/archie-core/internal/store" ) @@ -43,15 +44,17 @@ func newRemoteTestServer(t *testing.T) *Server { t.Fatalf("listen: %v", err) } server := grpc.NewServer() - // The same concrete *store.Store satisfies every interface the server's - // per-surface accessors need; leaving one nil makes ListMappings/ListBindings - // fail closed with err*Unavailable, which is the real composition's wiring. + // Two stores, as in the real composition: the task store serves the task + // surfaces, the event-capture store serves captures/mappings/bindings. + // Leaving one nil makes ListMappings/ListBindings fail closed with + // err*Unavailable, which is also the real wiring. + eda := edastore.OpenTest(t) staterpc.RegisterServer(server, staterpc.Deps{ Tasks: st, - Captures: st, - Mappings: st, - Bindings: st, - BindingDispatcher: st, + Captures: eda, + Mappings: eda, + Bindings: eda, + BindingDispatcher: eda, BindingTaskCreator: st, Log: slog.New(slog.DiscardHandler), }) diff --git a/proto/state/v1/state.proto b/proto/state/v1/state.proto index 891e27728..c24340147 100644 --- a/proto/state/v1/state.proto +++ b/proto/state/v1/state.proto @@ -231,7 +231,7 @@ message Task { int64 watch_comment_id = 20; string source = 21; string identity = 22; - int64 binding_id = 23; + string binding_id = 23; int64 binding_version = 24; google.protobuf.Timestamp created_at = 25; google.protobuf.Timestamp updated_at = 26; @@ -280,7 +280,7 @@ message Event { // CapturedEvent mirrors internal/store.CapturedEvent. message CapturedEvent { - int64 id = 1; + string id = 1; google.protobuf.Timestamp received_at = 2; string source = 3; string remote_addr = 4; @@ -300,7 +300,7 @@ message MappingField { // Mapping mirrors internal/domain/mapping.Mapping. message Mapping { - int64 id = 1; + string id = 1; string name = 2; string source_hint = 3; repeated MappingField fields = 4; @@ -315,10 +315,10 @@ message BindingMatcher { // Binding mirrors internal/domain/binding.Binding. message Binding { - int64 id = 1; + string id = 1; string name = 2; BindingMatcher matcher = 3; - int64 mapping_id = 4; + string mapping_id = 4; string workflow = 5; string owner = 6; string repo = 7; @@ -610,7 +610,7 @@ message InsertCaptureRequest { int64 retention_seconds = 2; int64 max_events = 3; } -message InsertCaptureResponse { int64 id = 1; } +message InsertCaptureResponse { string id = 1; } message ListCapturesRequest { int64 limit = 1; } message ListCapturesResponse { repeated CapturedEvent captures = 1; } @@ -619,9 +619,9 @@ message StreamCapturesRequest { int64 limit = 1; } message StreamCapturesResponse { CapturedEvent capture = 1; } message InsertMappingRequest { Mapping mapping = 1; } -message InsertMappingResponse { int64 id = 1; } +message InsertMappingResponse { string id = 1; } -message GetMappingRequest { int64 id = 1; } +message GetMappingRequest { string id = 1; } message GetMappingResponse { Mapping mapping = 1; bool found = 2; @@ -633,13 +633,13 @@ message ListMappingsResponse { repeated Mapping mappings = 1; } message UpdateMappingRequest { Mapping mapping = 1; } message UpdateMappingResponse {} -message DeleteMappingRequest { int64 id = 1; } +message DeleteMappingRequest { string id = 1; } message DeleteMappingResponse {} message InsertBindingRequest { Binding binding = 1; } -message InsertBindingResponse { int64 id = 1; } +message InsertBindingResponse { string id = 1; } -message GetBindingRequest { int64 id = 1; } +message GetBindingRequest { string id = 1; } message GetBindingResponse { Binding binding = 1; bool found = 2; @@ -651,19 +651,20 @@ message ListBindingsResponse { repeated Binding bindings = 1; } message UpdateBindingRequest { Binding binding = 1; } message UpdateBindingResponse {} -message DeleteBindingRequest { int64 id = 1; } +message DeleteBindingRequest { string id = 1; } message DeleteBindingResponse {} -message ApproveBindingRequest { int64 id = 1; } +message ApproveBindingRequest { string id = 1; } message ApproveBindingResponse {} message ArmedBindingsForSourceRequest { string source = 1; } message ArmedBindingsForSourceResponse { repeated Binding bindings = 1; } message RecordDispatchRequest { - int64 binding_id = 1; + string binding_id = 1; int64 binding_version = 2; - int64 capture_id = 3; + string capture_id = 3; + // task_id addresses the SQLite-owned task tables, which are not migrating. int64 task_id = 4; } message RecordDispatchResponse {} @@ -698,7 +699,7 @@ message EnqueueBindingTaskRequest { string body = 4; string workflow = 5; string identity = 6; - int64 binding_id = 7; + string binding_id = 7; int64 binding_version = 8; } message EnqueueBindingTaskResponse { Task task = 1; } From a4d772816c6f7717610235d8abd17645d7b03d39 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Wed, 23 Sep 2026 02:53:39 +1000 Subject: [PATCH 09/20] feat(scheduling): implement cron next-run arithmetic via pocketbase tools/cron --- internal/domain/scheduling/schedule.go | 91 ++++++-- internal/domain/scheduling/schedule_test.go | 229 ++++++++++++++++++++ 2 files changed, 302 insertions(+), 18 deletions(-) create mode 100644 internal/domain/scheduling/schedule_test.go diff --git a/internal/domain/scheduling/schedule.go b/internal/domain/scheduling/schedule.go index 25d044645..573e7ebb6 100644 --- a/internal/domain/scheduling/schedule.go +++ b/internal/domain/scheduling/schedule.go @@ -4,6 +4,8 @@ import ( "encoding/json" "fmt" "time" + + "github.com/pocketbase/pocketbase/tools/cron" ) // Duration is a schedule interval as the document writes it: the human string @@ -77,21 +79,20 @@ type Schedule struct { Start time.Time `json:"start"` } -// Schedule kinds the store recognises. Slice 1 fully implements -// Interval; Once is accepted at Create / fires at Due exactly once, but -// has no next-run arithmetic for MarkRun (one-shot by definition). Cron -// is parsed at Create but has no firing arithmetic yet — Slice 2 will -// add the parser. MarkRun on a kind this build does not implement -// returns ErrScheduleUnsupported rather than silently advancing next_run. +// Schedule kinds the store recognises. Interval and Cron have full +// next-run arithmetic; Once is accepted at Create / fires at Due exactly +// once, but has no next-run arithmetic for MarkRun (one-shot by +// definition). MarkRun on a kind this build does not implement returns +// ErrScheduleUnsupported rather than silently advancing next_run. const ( // ScheduleInterval fires every Interval after the last successful run // (or after Start, if never run). The simplest recurring schedule. ScheduleInterval = "interval" // ScheduleCron fires at the next moment matching a standard 5-field - // cron expression after the last successful run. Slice 2 will add - // the parser; until then, MarkRun on a cron job returns - // ErrScheduleUnsupported. + // cron expression (or one of the @ macros) after the last successful + // run, computed in the zone of that run time. Expressions are parsed + // with pocketbase tools/cron. ScheduleCron = "cron" // ScheduleOnce fires once at the configured At time. Subsequent @@ -110,8 +111,8 @@ func (s Schedule) Validate() error { return fmt.Errorf("%w: interval must be positive, got %v", ErrInvalidSpec, s.Interval) } case ScheduleCron: - if s.Cron == "" { - return fmt.Errorf("%w: cron expression must be non-empty", ErrInvalidSpec) + if _, err := parseCronSpec(s.Cron); err != nil { + return err } case ScheduleOnce: if s.At == nil { @@ -136,8 +137,8 @@ func (s Schedule) resolve() Schedule { // firstRun returns when the job should fire the very first time it has // never run before. For Interval, it is Start + Interval (or now + Interval // when Start is zero, so a freshly-created job with no Start still has a -// finite next-run). For Once, it is At. For Cron, it is left as a future -// responsibility — Slice 1 will not synthesise a value it cannot honour. +// finite next-run). For Once, it is At. For Cron, it is the first matching +// minute strictly after Start (or after now when Start is zero). func (s Schedule) firstRun(now time.Time) (time.Time, error) { switch s.resolve().Kind { case ScheduleInterval: @@ -151,12 +152,62 @@ func (s Schedule) firstRun(now time.Time) (time.Time, error) { } return *s.At, nil case ScheduleCron: - return time.Time{}, fmt.Errorf("%w: cron parsing not yet implemented (slice 2)", ErrScheduleUnsupported) + sched, err := parseCronSpec(s.Cron) + if err != nil { + return time.Time{}, err + } + anchor := now + if !s.Start.IsZero() { + anchor = s.Start + } + return nextCronRun(sched, anchor) default: return time.Time{}, fmt.Errorf("%w: unknown kind %q", ErrScheduleUnsupported, s.Kind) } } +// parseCronSpec parses expr with pocketbase tools/cron — the project's +// chosen cron dialect: standard 5-field expressions plus the @yearly, +// @monthly, @weekly, @daily and @hourly macros — and wraps any parse +// failure in ErrInvalidSpec so callers need only the one sentinel. +func parseCronSpec(expr string) (*cron.Schedule, error) { + sched, err := cron.NewSchedule(expr) + if err != nil { + return nil, fmt.Errorf("%w: cron expression %q: %w", ErrInvalidSpec, expr, err) + } + return sched, nil +} + +// cronHorizon bounds the next-run scan. A satisfiable expression always +// fires well within it (the sparsest realistic case, Feb 29, recurs every +// four years); only a spec that can never fire — e.g. "0 0 30 2 *", which +// the parser's field ranges cannot reject — exhausts the scan. +const cronHorizon = 5 * 365 * 24 * time.Hour + +// nextCronRun returns the first wall-clock minute in the location of from +// at which sched is due, strictly after the minute containing from — the +// minute that just fired is never its own successor. Whole months and +// hours that cannot match are skipped, so even a sparse expression costs +// a handful of iterations per year scanned. +func nextCronRun(sched *cron.Schedule, from time.Time) (time.Time, error) { + t := from.Truncate(time.Minute).Add(time.Minute) + for limit := from.Add(cronHorizon); t.Before(limit); { + if sched.IsDue(cron.NewMoment(t)) { + return t, nil + } + if _, ok := sched.Months[int(t.Month())]; !ok { + t = time.Date(t.Year(), t.Month(), 1, 0, 0, 0, 0, t.Location()).AddDate(0, 1, 0) + continue + } + if _, ok := sched.Hours[t.Hour()]; !ok { + t = t.Add(time.Hour).Truncate(time.Hour) + continue + } + t = t.Add(time.Minute) + } + return time.Time{}, fmt.Errorf("%w: cron expression does not match any minute within the next five years", ErrInvalidSpec) +} + func (s Schedule) FirstRun(now time.Time) (time.Time, error) { return s.firstRun(now) } func (s Schedule) NextRun(lastRun time.Time) (time.Time, error) { return s.nextRun(lastRun) } @@ -165,9 +216,9 @@ func (s Schedule) Resolved() Schedule { return s.resolve() } // nextRun returns when the job should fire after a successful run at // lastRun. It is the engine-side counterpart to firstRun: every MarkRun -// advances next_run through this function. Interval is fully implemented; -// Once and Cron deliberately fail so a silently-wrong value cannot reach -// the ticker. +// advances next_run through this function. Interval and Cron are fully +// implemented; Once deliberately fails (there is no recurring next run) +// so a silently-wrong value cannot reach the ticker. func (s Schedule) nextRun(lastRun time.Time) (time.Time, error) { switch s.resolve().Kind { case ScheduleInterval: @@ -175,7 +226,11 @@ func (s Schedule) nextRun(lastRun time.Time) (time.Time, error) { case ScheduleOnce: return time.Time{}, fmt.Errorf("%w: once schedule has no next run", ErrScheduleUnsupported) case ScheduleCron: - return time.Time{}, fmt.Errorf("%w: cron arithmetic not yet implemented (slice 2)", ErrScheduleUnsupported) + sched, err := parseCronSpec(s.Cron) + if err != nil { + return time.Time{}, err + } + return nextCronRun(sched, lastRun) default: return time.Time{}, fmt.Errorf("%w: unknown kind %q", ErrScheduleUnsupported, s.Kind) } diff --git a/internal/domain/scheduling/schedule_test.go b/internal/domain/scheduling/schedule_test.go new file mode 100644 index 000000000..3707024fc --- /dev/null +++ b/internal/domain/scheduling/schedule_test.go @@ -0,0 +1,229 @@ +package scheduling + +import ( + "errors" + "testing" + "time" +) + +// testZone is a fixed +02:00 zone: explicit, DST-free, and distinct from UTC +// so any arithmetic that silently computes in UTC fails. Every schedule test +// builds its times here, making the suite independent of the host's TZ. +var testZone = time.FixedZone("TEST", 2*60*60) + +// zoned builds a wall-clock time in the fixed test zone. +func zoned(year int, month time.Month, day, hour, minute, sec int) time.Time { + return time.Date(year, month, day, hour, minute, sec, 0, testZone) +} + +func TestScheduleCronNextRun(t *testing.T) { + tests := []struct { + name string + expr string + lastRun time.Time + want time.Time + }{ + { + name: "minute later in the same hour", + expr: "30 14 * * *", + lastRun: zoned(2026, 8, 24, 13, 0, 0), + want: zoned(2026, 8, 24, 14, 30, 0), + }, + { + name: "hour boundary skips the remainder of the hour", + expr: "*/15 * * * *", + lastRun: zoned(2026, 8, 24, 12, 7, 0), + want: zoned(2026, 8, 24, 12, 15, 0), + }, + { + name: "run exactly on a matching minute advances to the next one", + expr: "0 * * * *", + lastRun: zoned(2026, 8, 24, 12, 0, 0), + want: zoned(2026, 8, 24, 13, 0, 0), + }, + { + name: "across the hour boundary", + expr: "5,35 * * * *", + lastRun: zoned(2026, 8, 24, 12, 36, 0), + want: zoned(2026, 8, 24, 13, 5, 0), + }, + { + name: "range with step across hours", + expr: "0 9-17/2 * * *", + lastRun: zoned(2026, 8, 24, 10, 0, 0), + want: zoned(2026, 8, 24, 11, 0, 0), + }, + { + name: "across midnight", + expr: "0 0 * * *", + lastRun: zoned(2026, 8, 24, 23, 59, 0), + want: zoned(2026, 8, 25, 0, 0, 0), + }, + { + name: "across the month boundary", + expr: "0 0 1 * *", + lastRun: zoned(2026, 8, 31, 12, 0, 0), + want: zoned(2026, 9, 1, 0, 0, 0), + }, + { + name: "weekday rolls to next Monday", + expr: "0 0 * * 1", + lastRun: zoned(2026, 8, 21, 10, 0, 0), + want: zoned(2026, 8, 24, 0, 0, 0), + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + s := Schedule{Kind: ScheduleCron, Cron: tc.expr} + got, err := s.NextRun(tc.lastRun) + if err != nil { + t.Fatalf("NextRun(%v): %v", tc.lastRun, err) + } + if !got.Equal(tc.want) { + t.Fatalf("NextRun(%v) = %v (%s), want %v (%s)", + tc.lastRun, got, got.Format("2006-01-02 15:04:05 MST"), + tc.want, tc.want.Format("2006-01-02 15:04:05 MST")) + } + }) + } +} + +func TestScheduleCronNextRunMacros(t *testing.T) { + // lastRun is Monday 2026-08-24 12:15:30 TEST. + lastRun := zoned(2026, 8, 24, 12, 15, 30) + tests := []struct { + name string + expr string + want time.Time + }{ + {name: "@hourly", expr: "@hourly", want: zoned(2026, 8, 24, 13, 0, 0)}, + {name: "@daily", expr: "@daily", want: zoned(2026, 8, 25, 0, 0, 0)}, + {name: "@midnight", expr: "@midnight", want: zoned(2026, 8, 25, 0, 0, 0)}, + {name: "@weekly", expr: "@weekly", want: zoned(2026, 8, 30, 0, 0, 0)}, + {name: "@monthly", expr: "@monthly", want: zoned(2026, 9, 1, 0, 0, 0)}, + {name: "@yearly", expr: "@yearly", want: zoned(2027, 1, 1, 0, 0, 0)}, + {name: "@annually", expr: "@annually", want: zoned(2027, 1, 1, 0, 0, 0)}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + s := Schedule{Kind: ScheduleCron, Cron: tc.expr} + got, err := s.NextRun(lastRun) + if err != nil { + t.Fatalf("NextRun(%v): %v", lastRun, err) + } + if !got.Equal(tc.want) { + t.Fatalf("NextRun(%v) with %q = %v, want %v", lastRun, tc.expr, got, tc.want) + } + }) + } +} + +func TestScheduleCronComputesInThePassedZone(t *testing.T) { + s := Schedule{Kind: ScheduleCron, Cron: "0 6 * * *"} + + // lastRun 05:30 in +02:00: 06:00 local is 04:00 UTC. + got, err := s.NextRun(zoned(2026, 8, 24, 5, 30, 0)) + if err != nil { + t.Fatalf("NextRun: %v", err) + } + if want := time.Date(2026, 8, 24, 4, 0, 0, 0, time.UTC); !got.Equal(want) { + t.Fatalf("fixed-zone NextRun = %v (%s), want %v UTC", got, got.Format("15:04 MST"), want) + } + + // The same wall-clock spec anchored in UTC must land on 06:00 UTC instead. + got, err = s.NextRun(time.Date(2026, 8, 24, 5, 30, 0, 0, time.UTC)) + if err != nil { + t.Fatalf("NextRun: %v", err) + } + if want := time.Date(2026, 8, 24, 6, 0, 0, 0, time.UTC); !got.Equal(want) { + t.Fatalf("UTC NextRun = %v, want %v UTC", got, want) + } +} + +func TestScheduleCronFirstRun(t *testing.T) { + t.Run("anchored on Start", func(t *testing.T) { + s := Schedule{ + Kind: ScheduleCron, + Cron: "0 12 * * *", + Start: zoned(2026, 8, 24, 9, 0, 0), + } + want := zoned(2026, 8, 24, 12, 0, 0) + got, err := s.FirstRun(time.Date(2026, 8, 24, 6, 0, 0, 0, testZone)) + if err != nil { + t.Fatalf("FirstRun: %v", err) + } + if !got.Equal(want) { + t.Fatalf("FirstRun = %v, want %v", got, want) + } + }) + + t.Run("zero Start anchors on now", func(t *testing.T) { + s := Schedule{Kind: ScheduleCron, Cron: "0 12 * * *"} + now := zoned(2026, 8, 24, 12, 30, 0) + want := zoned(2026, 8, 25, 12, 0, 0) + got, err := s.FirstRun(now) + if err != nil { + t.Fatalf("FirstRun: %v", err) + } + if !got.Equal(want) { + t.Fatalf("FirstRun(%v) = %v, want %v", now, got, want) + } + }) +} + +func TestScheduleCronValidate(t *testing.T) { + t.Run("empty cron is rejected", func(t *testing.T) { + err := Schedule{Kind: ScheduleCron}.Validate() + if !errors.Is(err, ErrInvalidSpec) { + t.Fatalf("Validate() = %v, want ErrInvalidSpec", err) + } + }) + + t.Run("valid expressions pass", func(t *testing.T) { + for _, expr := range []string{"*/15 * * * *", "0 9-17/2 * * *", "@daily", "@yearly"} { + if err := (Schedule{Kind: ScheduleCron, Cron: expr}).Validate(); err != nil { + t.Fatalf("Validate(%q) = %v, want nil", expr, err) + } + } + }) +} + +func TestScheduleCronInvalidExpressions(t *testing.T) { + tests := []struct { + name string + expr string + }{ + {name: "minute out of range", expr: "61 * * * *"}, + {name: "hour out of range", expr: "* 24 * * *"}, + {name: "day of month out of range", expr: "0 0 32 * *"}, + {name: "month out of range", expr: "0 0 * 13 *"}, + {name: "day of week out of range", expr: "0 0 * * 7"}, + {name: "too few fields", expr: "* * * *"}, + {name: "too many fields", expr: "* * * * * *"}, + {name: "non-numeric field", expr: "banana * * * *"}, + {name: "zero step", expr: "*/0 * * * *"}, + {name: "empty", expr: ""}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + s := Schedule{Kind: ScheduleCron, Cron: tc.expr} + + if err := s.Validate(); !errors.Is(err, ErrInvalidSpec) { + t.Errorf("Validate() = %v, want ErrInvalidSpec", err) + } + + _, err := s.FirstRun(zoned(2026, 8, 24, 12, 0, 0)) + if !errors.Is(err, ErrInvalidSpec) { + t.Errorf("FirstRun err = %v, want ErrInvalidSpec", err) + } + + _, err = s.NextRun(zoned(2026, 8, 24, 12, 0, 0)) + if !errors.Is(err, ErrInvalidSpec) { + t.Errorf("NextRun err = %v, want ErrInvalidSpec", err) + } + }) + } +} From 2df53038ab682b936f6d3ac31d9716d8fd458dd5 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Wed, 23 Sep 2026 02:56:27 +1000 Subject: [PATCH 10/20] docs(scheduling): pin the pocketbase AND of day-of-month and day-of-week tools/cron ANDs the two day fields where Vixie cron ORs them, so 0 0 1 * 1 means Monday the 1st, not the 1st or any Monday. State the deviation in the ScheduleCron comment and pin it with a test. --- internal/domain/scheduling/schedule.go | 8 +++++--- internal/domain/scheduling/schedule_test.go | 8 ++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/internal/domain/scheduling/schedule.go b/internal/domain/scheduling/schedule.go index 573e7ebb6..787f02245 100644 --- a/internal/domain/scheduling/schedule.go +++ b/internal/domain/scheduling/schedule.go @@ -89,10 +89,12 @@ const ( // (or after Start, if never run). The simplest recurring schedule. ScheduleInterval = "interval" - // ScheduleCron fires at the next moment matching a standard 5-field - // cron expression (or one of the @ macros) after the last successful + // ScheduleCron fires at the next moment matching a 5-field cron + // expression (or one of the @ macros) after the last successful // run, computed in the zone of that run time. Expressions are parsed - // with pocketbase tools/cron. + // with pocketbase tools/cron, which ANDs day-of-month with day-of-week + // instead of Vixie cron's OR: "0 0 1 * 1" means only Monday the 1st, + // not "the 1st or any Monday". ScheduleCron = "cron" // ScheduleOnce fires once at the configured At time. Subsequent diff --git a/internal/domain/scheduling/schedule_test.go b/internal/domain/scheduling/schedule_test.go index 3707024fc..44734b435 100644 --- a/internal/domain/scheduling/schedule_test.go +++ b/internal/domain/scheduling/schedule_test.go @@ -35,6 +35,14 @@ func TestScheduleCronNextRun(t *testing.T) { lastRun: zoned(2026, 8, 24, 12, 7, 0), want: zoned(2026, 8, 24, 12, 15, 0), }, + { + // pins the pocketbase AND semantics: Vixie cron would give the + // next Monday (2026-08-31); tools/cron gives Monday the 1st. + name: "dual-restricted day-of-month and day-of-week AND, not Vixie OR", + expr: "0 0 1 * 1", + lastRun: zoned(2026, 8, 24, 12, 15, 30), + want: zoned(2027, 2, 1, 0, 0, 0), + }, { name: "run exactly on a matching minute advances to the next one", expr: "0 * * * *", From 62bda28be676159e6247cd6cca59e53da9e87d58 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Wed, 23 Sep 2026 02:57:45 +1000 Subject: [PATCH 11/20] feat(edastore): announce binding and mapping edits on the activity stream Each successful write on the State Store's PocketBase bindings/mappings tables emits one event (binding_changed / mapping_changed) through a nil-safe Notify func injected at composition, so a dashboard edit is visible without a manual refresh -- the same task-DB event path capture arrival uses. Nil notifier keeps every existing constructor silent. --- internal/events/events.go | 9 + .../edastore/contracts_binding.go | 33 ++- internal/infrastructure/edastore/edastore.go | 10 +- internal/infrastructure/edastore/notify.go | 26 ++ .../infrastructure/edastore/notify_test.go | 270 ++++++++++++++++++ 5 files changed, 342 insertions(+), 6 deletions(-) create mode 100644 internal/infrastructure/edastore/notify.go create mode 100644 internal/infrastructure/edastore/notify_test.go diff --git a/internal/events/events.go b/internal/events/events.go index 2c8010f02..6676faf9a 100644 --- a/internal/events/events.go +++ b/internal/events/events.go @@ -105,6 +105,15 @@ const ( // trigger. data: session, channel. KindTurnCompleted = "turn_completed" + // EDA edit activity (bindings and mappings on the captures store). Each + // successful write on the State Store's PocketBase tables emits one + // event through the same task-DB path capture arrival uses, so a + // dashboard edit is visible on the activity stream without a manual + // refresh. data: id (the edited record), action (create, update, + // approve, delete). + KindBindingChanged = "binding_changed" + KindMappingChanged = "mapping_changed" + // KindUpdateReport carries the phase-2 outcome of a dashboard-initiated // update -- whether the restarted daemon came back up healthy and on // the version it claimed, relayed once on the boot that finds the diff --git a/internal/infrastructure/edastore/contracts_binding.go b/internal/infrastructure/edastore/contracts_binding.go index cc61cb4ad..30cb32268 100644 --- a/internal/infrastructure/edastore/contracts_binding.go +++ b/internal/infrastructure/edastore/contracts_binding.go @@ -10,6 +10,7 @@ import ( "github.com/samcharles93/archie-core/internal/domain/binding" "github.com/samcharles93/archie-core/internal/domain/mapping" + "github.com/samcharles93/archie-core/internal/events" ) // --- mappings --- @@ -54,6 +55,7 @@ func (s *Store) InsertMapping(_ context.Context, m mapping.Mapping) (string, err if err := s.app.Save(r); err != nil { return "", fmt.Errorf("edastore: insert mapping: %w", err) } + s.notifyWrite(events.KindMappingChanged, "mapping", "create", r.Id) return r.Id, nil } @@ -96,7 +98,11 @@ func (s *Store) UpdateMapping(_ context.Context, m mapping.Mapping) error { if err := s.applyMapping(r, m); err != nil { return err } - return s.app.Save(r) + if err := s.app.Save(r); err != nil { + return err + } + s.notifyWrite(events.KindMappingChanged, "mapping", "update", m.ID) + return nil } func (s *Store) DeleteMapping(_ context.Context, id string) error { @@ -107,7 +113,11 @@ func (s *Store) DeleteMapping(_ context.Context, id string) error { if r == nil { return ErrMappingNotFound } - return s.app.Delete(r) + if err := s.app.Delete(r); err != nil { + return err + } + s.notifyWrite(events.KindMappingChanged, "mapping", "delete", id) + return nil } // --- bindings --- @@ -189,6 +199,7 @@ func (s *Store) InsertBinding(ctx context.Context, b binding.Binding) (string, e return "", fmt.Errorf("edastore: insert binding: %w", err) } _ = ctx + s.notifyWrite(events.KindBindingChanged, "binding", "create", r.Id) return r.Id, nil } @@ -262,7 +273,11 @@ func (s *Store) UpdateBinding(_ context.Context, b binding.Binding) error { } r.Set("status", string(binding.StatusPendingApproval)) r.Set("version", r.GetInt("version")+1) - return s.app.Save(r) + if err := s.app.Save(r); err != nil { + return err + } + s.notifyWrite(events.KindBindingChanged, "binding", "update", b.ID) + return nil } func (s *Store) DeleteBinding(_ context.Context, id string) error { @@ -273,7 +288,11 @@ func (s *Store) DeleteBinding(_ context.Context, id string) error { if r == nil { return ErrBindingNotFound } - return s.app.Delete(r) + if err := s.app.Delete(r); err != nil { + return err + } + s.notifyWrite(events.KindBindingChanged, "binding", "delete", id) + return nil } // ApproveBinding is the only transition that arms a binding. It moves @@ -303,7 +322,11 @@ func (s *Store) ApproveBinding(_ context.Context, id string) error { return ErrBindingTransition } r.Set("status", string(binding.StatusArmed)) - return s.app.Save(r) + if err := s.app.Save(r); err != nil { + return err + } + s.notifyWrite(events.KindBindingChanged, "binding", "approve", id) + return nil } // --- dispatch ledgers --- diff --git a/internal/infrastructure/edastore/edastore.go b/internal/infrastructure/edastore/edastore.go index 15d2922ff..9c4c6c86e 100644 --- a/internal/infrastructure/edastore/edastore.go +++ b/internal/infrastructure/edastore/edastore.go @@ -29,6 +29,7 @@ import ( _ "modernc.org/sqlite" "github.com/samcharles93/archie-core/internal/domain/storecontract" + "github.com/samcharles93/archie-core/internal/events" ) // Collection names. They are unprefixed because this store owns its database; @@ -71,6 +72,12 @@ type Config struct { // Cipher encrypts binding secrets at rest. Nil keeps them plaintext, // which is the behaviour that predates the option. Cipher BindingCipher + // Notify receives one event per successful write on the bindings and + // mappings tables, so an operator edit surfaces on the dashboard's + // activity stream exactly like capture arrival does. Nil keeps writes + // silent, which is the behaviour that predates the option and what + // every existing constructor gets. + Notify func(events.Event) // DBPath is the SQLite file PocketBase owns for these collections. DBPath string // DataDir holds PocketBase's auxiliary database and settings. @@ -81,6 +88,7 @@ type Config struct { type Store struct { app *pocketbase.PocketBase cipher BindingCipher + notify func(events.Event) } // Open bootstraps the store and ensures its collections exist. @@ -110,7 +118,7 @@ func Open(cfg Config) (*Store, error) { if err := ensureCollections(app); err != nil { return nil, err } - return &Store{app: app, cipher: cfg.Cipher}, nil + return &Store{app: app, cipher: cfg.Cipher, notify: cfg.Notify}, nil } // App exposes the PocketBase application so one process can serve this store diff --git a/internal/infrastructure/edastore/notify.go b/internal/infrastructure/edastore/notify.go new file mode 100644 index 000000000..f8ecec7f3 --- /dev/null +++ b/internal/infrastructure/edastore/notify.go @@ -0,0 +1,26 @@ +package edastore + +import "github.com/samcharles93/archie-core/internal/events" + +// notifyWrite announces one successful write on the bindings or mappings +// tables. It rides the same activity-event path capture arrival uses: the +// State Store's event insert feeds the webui pump, which broadcasts to every +// connected browser, so an operator edit is visible without a manual +// refresh. The data carries the record id and action; the row itself stays +// the only source of the full record, and consumers refetch through the +// authenticated API -- the event is an invalidation signal, not a payload. +// +// A nil notifier is a silent no-op: stores composed without one (tests, +// tooling) keep working unchanged, mirroring the nil Publish on +// captureintake.Receiver. +func (s *Store) notifyWrite(kind, subject, action, id string) { + if s.notify == nil { + return + } + past := map[string]string{"create": "created", "update": "updated", "approve": "approved", "delete": "deleted"}[action] + s.notify(events.Event{ + Kind: kind, + Detail: subject + " " + past, + Data: map[string]any{"id": id, "action": action}, + }) +} diff --git a/internal/infrastructure/edastore/notify_test.go b/internal/infrastructure/edastore/notify_test.go new file mode 100644 index 000000000..b85cf58a8 --- /dev/null +++ b/internal/infrastructure/edastore/notify_test.go @@ -0,0 +1,270 @@ +package edastore + +import ( + "errors" + "testing" + + "github.com/samcharles93/archie-core/internal/domain/binding" + "github.com/samcharles93/archie-core/internal/domain/mapping" + "github.com/samcharles93/archie-core/internal/events" +) + +// recordingNotifier collects every event the store emits so a test can +// assert on the exact sequence, mirroring captureintake's recordingPublisher. +type recordingNotifier struct { + got []events.Event +} + +func (r *recordingNotifier) notify(e events.Event) { r.got = append(r.got, e) } + +func (r *recordingNotifier) reset() { r.got = nil } + +// notifiedStore is an OpenTest store with the notifier wired, the way +// composition does it through Config.Notify. +func notifiedStore(t *testing.T) (*Store, *recordingNotifier) { + t.Helper() + s := OpenTest(t) + n := &recordingNotifier{} + s.notify = n.notify + return s, n +} + +func TestWriteMethodsNotifyOnSuccess(t *testing.T) { + b := func(source string) binding.Binding { + return binding.Binding{ + Name: "webhook to workflow", + Matcher: binding.Matcher{Source: source}, + Workflow: "triage", + } + } + + t.Run("InsertBinding", func(t *testing.T) { + s, n := notifiedStore(t) + id, err := s.InsertBinding(t.Context(), b("src-insert")) + if err != nil { + t.Fatalf("InsertBinding() error = %v", err) + } + if len(n.got) != 1 { + t.Fatalf("InsertBinding notified %d times, want 1: %+v", len(n.got), n.got) + } + assertEditEvent(t, n.got[0], events.KindBindingChanged, "create", id) + }) + + t.Run("UpdateBinding", func(t *testing.T) { + s, n := notifiedStore(t) + id := insertBinding(t, s, bindingInput{Name: "before", Source: "src-update", Workflow: "triage"}) + n.reset() + bb := b("src-update") + bb.ID = id + bb.Workflow = "escalate" + if err := s.UpdateBinding(t.Context(), bb); err != nil { + t.Fatalf("UpdateBinding() error = %v", err) + } + if len(n.got) != 1 { + t.Fatalf("UpdateBinding notified %d times, want 1: %+v", len(n.got), n.got) + } + assertEditEvent(t, n.got[0], events.KindBindingChanged, "update", id) + }) + + t.Run("ApproveBinding", func(t *testing.T) { + s, n := notifiedStore(t) + id := insertBinding(t, s, bindingInput{Name: "draft", Source: "src-approve", Workflow: "triage"}) + bb := b("src-approve") + bb.ID = id + // Approval requires pending_approval; the edit transition also fires. + if err := s.UpdateBinding(t.Context(), bb); err != nil { + t.Fatalf("UpdateBinding() error = %v", err) + } + n.reset() + if err := s.ApproveBinding(t.Context(), id); err != nil { + t.Fatalf("ApproveBinding() error = %v", err) + } + if len(n.got) != 1 { + t.Fatalf("ApproveBinding notified %d times, want 1: %+v", len(n.got), n.got) + } + assertEditEvent(t, n.got[0], events.KindBindingChanged, "approve", id) + }) + + t.Run("DeleteBinding", func(t *testing.T) { + s, n := notifiedStore(t) + id := insertBinding(t, s, bindingInput{Name: "doomed", Source: "src-delete", Workflow: "triage"}) + n.reset() + if err := s.DeleteBinding(t.Context(), id); err != nil { + t.Fatalf("DeleteBinding() error = %v", err) + } + if len(n.got) != 1 { + t.Fatalf("DeleteBinding notified %d times, want 1: %+v", len(n.got), n.got) + } + assertEditEvent(t, n.got[0], events.KindBindingChanged, "delete", id) + }) + + t.Run("InsertMapping", func(t *testing.T) { + s, n := notifiedStore(t) + id, err := s.InsertMapping(t.Context(), mapping.Mapping{Name: "payload-map"}) + if err != nil { + t.Fatalf("InsertMapping() error = %v", err) + } + if len(n.got) != 1 { + t.Fatalf("InsertMapping notified %d times, want 1: %+v", len(n.got), n.got) + } + assertEditEvent(t, n.got[0], events.KindMappingChanged, "create", id) + }) + + t.Run("UpdateMapping", func(t *testing.T) { + s, n := notifiedStore(t) + id, err := s.InsertMapping(t.Context(), mapping.Mapping{Name: "payload-map"}) + if err != nil { + t.Fatalf("InsertMapping() error = %v", err) + } + n.reset() + m, err := s.GetMapping(t.Context(), id) + if err != nil { + t.Fatalf("GetMapping() error = %v", err) + } + m.Name = "renamed" + if err := s.UpdateMapping(t.Context(), *m); err != nil { + t.Fatalf("UpdateMapping() error = %v", err) + } + if len(n.got) != 1 { + t.Fatalf("UpdateMapping notified %d times, want 1: %+v", len(n.got), n.got) + } + assertEditEvent(t, n.got[0], events.KindMappingChanged, "update", id) + }) + + t.Run("DeleteMapping", func(t *testing.T) { + s, n := notifiedStore(t) + id, err := s.InsertMapping(t.Context(), mapping.Mapping{Name: "doomed"}) + if err != nil { + t.Fatalf("InsertMapping() error = %v", err) + } + n.reset() + if err := s.DeleteMapping(t.Context(), id); err != nil { + t.Fatalf("DeleteMapping() error = %v", err) + } + if len(n.got) != 1 { + t.Fatalf("DeleteMapping notified %d times, want 1: %+v", len(n.got), n.got) + } + assertEditEvent(t, n.got[0], events.KindMappingChanged, "delete", id) + }) +} + +func assertEditEvent(t *testing.T, e events.Event, kind, action, id string) { + t.Helper() + if e.Kind != kind { + t.Errorf("event kind = %q, want %q", e.Kind, kind) + } + if e.Detail == "" { + t.Errorf("event detail is empty") + } + if got := e.Data["id"]; got != id { + t.Errorf("event data id = %v, want %q", got, id) + } + if got := e.Data["action"]; got != action { + t.Errorf("event data action = %v, want %q", got, action) + } +} + +func TestFailedWritesNotifyNothing(t *testing.T) { + s, n := notifiedStore(t) + + // Seed one binding and one mapping, then clear the seed-time events. + seedID := insertBinding(t, s, bindingInput{Name: "seed", Source: "src-seed", Workflow: "triage"}) + mid, err := s.InsertMapping(t.Context(), mapping.Mapping{Name: "seed-map"}) + if err != nil { + t.Fatalf("InsertMapping() error = %v", err) + } + n.reset() + + t.Run("UpdateBinding nonexistent", func(t *testing.T) { + bb := binding.Binding{ID: "missing", Matcher: binding.Matcher{Source: "src-other"}} + if err := s.UpdateBinding(t.Context(), bb); !errors.Is(err, ErrBindingNotFound) { + t.Fatalf("UpdateBinding() error = %v, want ErrBindingNotFound", err) + } + }) + t.Run("ApproveBinding refuses draft", func(t *testing.T) { + if err := s.ApproveBinding(t.Context(), seedID); !errors.Is(err, ErrBindingTransition) { + t.Fatalf("ApproveBinding() error = %v, want ErrBindingTransition", err) + } + }) + t.Run("InsertBinding overlaps source", func(t *testing.T) { + if _, err := s.InsertBinding(t.Context(), binding.Binding{ + Matcher: binding.Matcher{Source: "src-seed"}, + }); !errors.Is(err, ErrBindingOverlap) { + t.Fatalf("InsertBinding() error = %v, want ErrBindingOverlap", err) + } + }) + t.Run("DeleteBinding nonexistent", func(t *testing.T) { + if err := s.DeleteBinding(t.Context(), "missing"); !errors.Is(err, ErrBindingNotFound) { + t.Fatalf("DeleteBinding() error = %v, want ErrBindingNotFound", err) + } + }) + t.Run("UpdateMapping nonexistent", func(t *testing.T) { + if err := s.UpdateMapping(t.Context(), mapping.Mapping{ID: "missing"}); !errors.Is(err, ErrMappingNotFound) { + t.Fatalf("UpdateMapping() error = %v, want ErrMappingNotFound", err) + } + }) + t.Run("DeleteMapping nonexistent", func(t *testing.T) { + if err := s.DeleteMapping(t.Context(), "missing"); !errors.Is(err, ErrMappingNotFound) { + t.Fatalf("DeleteMapping() error = %v, want ErrMappingNotFound", err) + } + }) + _ = mid + + if len(n.got) != 0 { + t.Fatalf("failed writes notified %d times: %+v", len(n.got), n.got) + } +} + +// TestNilNotifierIsNoOp pins that a store composed without a notifier keeps +// every write path working: the existing constructors and tests never set +// one, so a nil-dereference there would break them all. +func TestNilNotifierIsNoOp(t *testing.T) { + s := OpenTest(t) + id, err := s.InsertBinding(t.Context(), binding.Binding{ + Name: "nil-safe", + Matcher: binding.Matcher{Source: "src-nil"}, + }) + if err != nil { + t.Fatalf("InsertBinding with nil notifier: error = %v", err) + } + bb, err := s.GetBinding(t.Context(), id) + if err != nil { + t.Fatalf("GetBinding() error = %v", err) + } + bb.Workflow = "escalate" + if err := s.UpdateBinding(t.Context(), *bb); err != nil { + t.Fatalf("UpdateBinding() error = %v", err) + } + if err := s.ApproveBinding(t.Context(), id); err != nil { + t.Fatalf("ApproveBinding() error = %v", err) + } + if err := s.DeleteBinding(t.Context(), id); err != nil { + t.Fatalf("DeleteBinding() error = %v", err) + } + if _, err := s.InsertMapping(t.Context(), mapping.Mapping{Name: "m"}); err != nil { + t.Fatalf("InsertMapping() error = %v", err) + } +} + +// TestOpenHonoursConfigNotify pins the composition surface: the wire-in sets +// Config.Notify, not the unexported field. +func TestOpenHonoursConfigNotify(t *testing.T) { + n := &recordingNotifier{} + dir := t.TempDir() + s, err := Open(Config{ + DBPath: t.TempDir() + "/eda.sqlite", + DataDir: dir, + Notify: n.notify, + }) + if err != nil { + t.Fatalf("Open() error = %v", err) + } + t.Cleanup(func() { _ = s.Close() }) + + if _, err := s.InsertMapping(t.Context(), mapping.Mapping{Name: "wired"}); err != nil { + t.Fatalf("InsertMapping() error = %v", err) + } + if len(n.got) != 1 { + t.Fatalf("store built from Config.Notify notified %d times, want 1: %+v", len(n.got), n.got) + } +} From 3292ef5117b3dd7d74566d671f471c8cf034dcd1 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Wed, 23 Sep 2026 02:59:03 +1000 Subject: [PATCH 12/20] fix(proto): record the deliberate int64->string break from the PocketBase migration The event-capture contract moved its record ids from the SQLite rowid (int64) to PocketBase's string record ids in place, on the same field numbers: 16 fields across CapturedEvent, Mapping, Binding, RecordDispatchRequest, EnqueueBindingTaskRequest, Task.binding_id and the Id-bearing request/response messages. buf breaking has flagged them since that commit landed, so task check has never passed on this branch. Both halves of the State Store contract ship from this repository together, so they cannot be versioned apart. Record the break narrowly: only FIELD_SAME_TYPE is relaxed, and only for state/v1/state.proto. Every other FILE rule still applies to that file and no other contract is affected. --- proto/buf.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/proto/buf.yaml b/proto/buf.yaml index f74da98a3..6d9f7222d 100644 --- a/proto/buf.yaml +++ b/proto/buf.yaml @@ -7,3 +7,15 @@ lint: breaking: use: - FILE + # The PocketBase migration moved the event-capture contract's record ids from + # the SQLite rowid (int64) to PocketBase's string record ids, in place, on the + # same field numbers: 16 fields across CapturedEvent, Mapping, Binding, + # RecordDispatchRequest, EnqueueBindingTaskRequest, Task.binding_id and the + # Id-bearing request/response messages. That is an intentional wire break, and + # both ends of the State Store contract ship from this repository together, so + # the two halves cannot be versioned apart. Only the type rule is relaxed, and + # only in this file: every other FILE breaking rule still applies here, and no + # other contract is affected. + ignore_only: + FIELD_SAME_TYPE: + - state/v1/state.proto From 05dfe00e31a6d09906f1e566e065ff12dc4fcaea Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Wed, 23 Sep 2026 02:59:35 +1000 Subject: [PATCH 13/20] chore(docs): regenerate dev-docs.json after the architecture doc move docs:artifact:check has failed on this branch since 024dfbbe deleted ARCHITECTURE.md and moved its normative content into docs/architecture; the generated dev-docs artifact was never regenerated, so task check could not pass. Regenerated from a clean tree (no untracked docs present). --- docs/data/generated/dev-docs.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/data/generated/dev-docs.json b/docs/data/generated/dev-docs.json index 4571c236c..2c85a1f3f 100644 --- a/docs/data/generated/dev-docs.json +++ b/docs/data/generated/dev-docs.json @@ -28,7 +28,7 @@ "url": "/dev/architecture/configuration/", "title": "Configuration", "section": "Architecture", - "body": "# Configuration\n\n**Status:** Approved; existing-type assignment is ongoing \n**Date:** 2026-07-28 \n**Tracking issue:** [#73](https://github.com/samcharles93/archie-core/issues/73)\n\n## Ownership\n\nConfiguration has two separate responsibilities:\n\n1. `internal/infrastructure/configuration` loads files, environment variables,\n secret references, overlays, and compatibility representations.\n2. Each domain and plugin owns the typed runtime settings required for its\n behaviour.\n\n`internal/app` translates external input into those settings, validates\ncross-capability combinations, and supplies each component only its own\nsettings.\n\nThe following are prohibited:\n\n- passing complete application configuration into a domain;\n- making a domain depend on TOML, YAML, environment-variable names, secret\n references, or a configuration library;\n- defining a central runtime configuration type imported across the system;\n- placing plugin options in an unrelated global configuration model.\n\n## Private input document\n\n`internal/infrastructure/configuration` MAY use a complete decoded document\nwhile discovering files, decoding sections, and applying overlays. It is an\ninput-boundary DTO only.\n\nThe document:\n\n- remains private to the configuration infrastructure;\n- is composed from separately defined feature sections;\n- is immediately translated by application composition;\n- is never returned as the application's runtime model;\n- is never passed through runtime components.\n\nExternal DTOs and domain runtime settings are separate types.\n\nPlugins supply external configuration definitions through a narrow extension\ncontract. The core loader MUST NOT gain a field or recognized filename for every\nplugin.\n\nInvalid configuration input MUST NOT replace active valid settings or terminate\nunaffected capabilities. Configuration changes use the candidate validation,\npromotion, audit, and recovery requirements in\n[safe-change-and-recovery.md](/dev/architecture/safe-change-and-recovery/).\n\n## Dissolution of `internal/config`\n\n`internal/config` has no target location. It MUST be dissolved and MUST NOT\nsurvive as a shared package, configuration domain, or renamed global model.\n\nIts loading behaviour moves to `internal/infrastructure/configuration`. Existing\ntypes and methods are reassigned by behaviour. Repository execution policy,\ndispatch policy, retry policy, ecosystem defaults, tool policy, memory settings,\nindexing settings, provider settings, and channel settings move to their\nrespective owners.\n\n## Repository settings\n\nThe existing `config.Repo` MUST NOT move intact into a repository domain or\nshared settings package. A managed repository is an application association:\n\n- work intake owns which repository references an identity monitors;\n- task execution owns gates, preflight, test protection, ecosystem behaviour,\n and change limits;\n- task lifecycle owns retry behaviour;\n- scheduling owns concurrency policy;\n- workspace management owns base branch and persistent-storage requirements.\n\nApplication composition correlates these settings by repository reference. No\ndomain receives a universal repository profile.\n\n## Documentation\n\nGenerated configuration reference describes external sections and their owning\ncapabilities. It MUST NOT establish a global Go configuration structure as the\npublic contract.\n\n## Value classification rule\n\nEvery value that controls behaviour MUST have an explicit owner and MUST be\nclassified as exactly one of:\n\n1. **Invariant** — changing it changes the meaning of the program, a wire\n protocol, or a domain concept. It may be a Go constant.\n2. **Runtime setting** — an operator, identity, domain, plugin, or deployment\n may legitimately choose a different value. It is typed configuration owned by\n the affected capability.\n3. **Policy value** — it expresses what is allowed, limited, routed, retried,\n protected, or gated. Its typed definition belongs to the owning domain and it\n is evaluated through the policy system.\n4. **Derived value** — it is computed from other settings and is not\n independently mutable.\n5. **Runtime state** — it changes as the application operates, such as the\n active model for a conversation. It is changed by a domain command and is not\n configuration merely because it can be changed.\n\nAn unclassified behavioural literal is a defect.\n\n`const` MUST NOT be used merely because a value currently has one default.\nConstants are reserved for actual invariants: enum discriminants, protocol\nversions, fixed compatibility tokens, and values whose alteration would change\nthe program's semantics rather than configure an installation.\n\nIn Go, a constant is a compile-time value and does not represent mutable runtime\nstorage. It generally has no address that can be observed or changed.\nArchitecturally, a constant therefore asserts that the value cannot change for\nthe lifetime of the application. Any value that may legitimately change while\nthe application is running, between application instances, or between\ndeployments is not a constant.\n\nA runtime setting may have a default, but the default is not its only access\npath. The owner exposes a typed settings structure, validation, and a\n`DefaultSettings` constructor or equivalent. Private constants used to implement\ndefaults do not turn those defaults into invariants and MUST NOT be consumed\ndirectly by runtime code.\n\nEach runtime setting and policy value MUST declare:\n\n- its owner;\n- type, units, validation constraints, and safe bounds;\n- default and derivation, if any;\n- whether it is startup-only, live-reloadable, or restart-required;\n- whether it may differ by installation, identity, repository, workflow,\n channel, or plugin instance;\n- whether it is secret, sensitive, or safe to report;\n- its current source, effective value, version, and last-known-good version;\n- the command and identity required to change it.\n\n## Current configuration migration ledger\n\nThis ledger records the current meanings hidden inside `internal/config`. It is\nthe source inventory for dissolving that package; rows are split where the\nexisting structure combines different owners.\n\n| Current values | Current location | Target owner and requirement |\n| ----------------------------------------------------------------------------------------------- | --------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- |\n| `WorkDir`, `DBPath` | root `Config` | Workspace/storage infrastructure settings. `DBPath` is the prefix for independent task and conversation SQLite files. Defaults derive from the platform data directory. |\n| `SkillsDir`, `PluginDir` | root `Config` | Skill and plugin infrastructure settings, independently typed. |\n| `PollInterval` | root `Config` | Work-intake scheduling setting; not a daemon-wide constant. |\n| `Label`, `BotUser`, `BotEmail` | root `Config` | Identity and dispatch associations. `BotEmail` may be derived from the selected forge account but remains overrideable. |\n| `MaxRetries` | root `Config`, `Repo` | Retry policy owned by the operation being retried. The duplicate global/repository representation must disappear. |\n| `DiffCapLines` | root `Config` | Agent-change policy value. |\n| `MaxSteps`, `WallClock`, `GateMaxFailures` | `Budgets` | Workflow-execution policy, with units and zero/unlimited semantics made explicit. |\n| `Class`, `APIKeyEnv`, `BaseURL` | `Provider` | Provider instance settings. Secret lookup becomes a `SecretRef`, not an environment variable name leaking into the provider domain. |\n| `Mode`, `Command`, `Env` | `LegacyAgent` | Decode-only compatibility input for the removed executor selector. Delete after the config migration window; it has no runtime settings owner. |\n| `Type`, `Host`, `Token` / `TokenEnv` (older form) | `Forge` | Forge adapter settings. The older `TokenEnv` input is accepted only at the loader boundary. |\n| `Trigger`, `AckReaction`, label names | `Dispatch` | Work-intake and forge presentation policy. Label vocabulary is configurable per forge/identity association. |\n| `Engine` | `MemoryConfig` | Memory engine selection. `internal/domain/memory` is the only engine family; per-engine settings remain owned and validated by that engine. |\n| `Name`, `Transport`, `Command`, `Args`, `WorkDir`, `URL` | `MCPServer` | One typed MCP server instance specification. Transport-specific variants must reject fields belonging to other variants. |\n| `MaxResultChars` | `ToolPolicy` | Tool-execution policy values, not generic daemon configuration. |\n| `URL`, `TokenEnv` | `NATSConfig` | NATS transport settings with a secret reference replacing the environment-name field. |\n| `Image`, `MaxConcurrency`, `MaxUptime`, `VolumeTTL`, `PullPolicy`, `Network` | `ContainerConfig` | Mandatory managed-worker infrastructure settings and resource policy. The legacy `enabled` input is decode-only. |\n| model mappings, email, webhook, Telegram | `ChatConfig` | Separate channel instance settings plus channel-routing associations; there is no chat-wide settings owner. |\n| `AllowedUserIDs`, `Token` / `TokenEnv` (older form), update commands | Telegram config | Telegram adapter settings and access policy. `Token` is the preferred `secret.SecretRef`; `TokenEnv` remains a backwards-compatible fallback. |\n| listen/relay addresses | email config | Email adapter instance settings. |\n| webhook target | `Notify` | Notification adapter instance settings. |\n| web listen address | `Web` | Web transport settings. |\n| `listen` under `[services.state]` and `[services.gateway]` | `ServiceConnection` | The bind address of the process that owns the service. Read only by that process, and defaulted to the address its clients dial so the pair cannot drift. |\n| index directory and database path | `Indexing` | Index infrastructure settings, derived from storage settings unless explicitly overridden. |\n| forge, dispatch, models, providers, containers, memory, tools, indexing and repositories | `IdentityConfig` | Existing composite duplicates unrelated settings. Replace it with associations from an `IdentityID` to independently owned configured instances. |\n| owner, name, base, gate, protection, ecosystem, preflight, storage, retry and concurrency | `Repo` | Split across work intake, workspace, workflow execution, policy, gate, and scheduling as described in Repository settings. |\n| `TaskConfig`, `TaskForge` | worker snapshot | Replace with the minimum immutable, versioned WorkflowExecution input assembled by the application. It is not a second configuration model. |\n| `Extra` and arbitrary maps | root/provider config | Allowed only at an extension boundary whose owning plugin/provider supplies schema and validation. Core configuration never consumes an untyped catch-all. |\n\n## Behavioural literals and incorrectly fixed values\n\nThe following current values affect useful behaviour but are not represented as\nowned typed settings or policy. They MUST be classified and migrated before the\nold configuration package is removed. The listed target is the architectural\ndecision; exact setting names are implementation details.\n\n| Capability | Values fixed in current code | Required classification |\n| ---------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |\n| Chat agent | maximum steps `8` in `cmd/archied` | Workflow/chat execution policy. |\n| Context compression | enabled, threshold `0.5`, context size `128000`, protect first `3` and last `20`, compression marker, token estimate of four characters per token | Model/context runtime settings. The marker is presentation; the estimate is provider/model strategy, not a universal truth. |\n| Tool guardrails | escalation thresholds `3`, `5`, `10`, `2` | Tool-use policy with named meanings and bounded defaults. |\n| Tool output | per-result cap `50000` characters, no aggregate turn cap, preview `500` characters | Tool-execution resource and presentation settings. Per-result spill/truncation remains; turn continuation is not character- or token-gated. |\n| Pairing/access | code TTL `1h`, maximum pending `3`, rate window `10m`, failures `5`, lockout `1h`, maximum lockout `24h` | Access policy. Code length/alphabet and salt length require an explicit security-constraint decision; they are not automatically valid constants. |\n| NATS daemon consumer | stream `ARCHIE_TASKS`, consumer `archie-daemon`, deduplication `2m`, poll `2s`, deliveries `3`, acknowledgement wait `5m`, inactive threshold `24h`, subjects | NATS topology settings plus delivery policy. Subject grammar or protocol headers may be invariants only if declared as versioned wire contracts. |\n| NATS worker consumer | stream `ARCHIE_TASKS`, consumer `archie-agent`, deduplication `2m`, poll `5s`, acknowledgement wait `30m`, deliveries `3`, inactive threshold `24h`, queue group and retry delay `1s` | Worker transport settings and delivery/retry policy. Shared stream defaults must have one owner rather than two copies. |\n| Worker readiness | readiness timeout `20s`, retry delay `250ms` | Worker lifecycle and retry policy. |\n| Execution/RPC | agent execution `30m`, task-run RPC `60s`, worktree RPC `15m` | Per-operation timeout policy. |\n| Indexing | build timeout `10m`, metadata timeout `5s` | Index operation policy. |\n| Plugin/tool cleanup | plugin rollback `10s`, tool-provider cleanup `10s`, MCP-provider cleanup `10s` | Lifecycle shutdown/rollback policy owned by each engine, with a shared policy vocabulary where appropriate. |\n| Memory read path | per-scope record limit `20`, rendered `\u003cmemory\u003e` block byte cap `8192` | Memory read policy owned by the engine family (`internal/domain/memory`). |\n| Built-in memory store | per-file size limit `100KiB`, query result limit `100` | Split into memory resource policy, input policy, and presentation settings. |\n| Memory safety scanner | compiled prompt-injection and sensitive-data patterns with fixed warn/block consequences | Versioned memory-domain policy definitions. Patterns and consequences must be inspectable and replaceable without editing scanner code. |\n| MCP stdio | initial backoff `500ms`, maximum backoff `30s`, unlimited default retries, send and shutdown `5s` | MCP transport retry, request, and lifecycle policy. Unlimited retries must be explicit rather than a zero-value accident. |\n| MCP SSE/HTTP | reconnect `1s` to `30s`, POST/request timeout `30s` | MCP transport settings and retry policy. |\n| Message framing | MCP frame `4MiB`, worker protocol frame `8MiB`, diagnostic `64KiB`, media/base64 `64MiB` | Resource/security policy unless a published wire protocol mandates the limit. Limits must be configurable only within safe bounds. |\n| Webhooks and web UI | webhook bind `0.0.0.0:8644`, webhook shutdown `5s`, web read-header `5s`, web shutdown `2s` | Transport instance and lifecycle settings. |\n| Telegram | typing refresh `4s`, draft interval `700ms`, update-action TTL `10m` | Telegram adapter presentation and lifecycle settings. |\n| Telegram limits | message length `4000` and callback/update prefixes | Record the upstream platform constraint and version it as an adapter compatibility limit; prefixes may remain wire constants when changing them would invalidate outstanding callbacks. |\n| Containers | stop and pull timeouts `10s` and `30s` | Container lifecycle policy. |\n| HTTP clients | feasibility notification timeout `10s`; other paths use `http.DefaultClient` | Adapter request policy. Every outbound client requires an explicit timeout profile. |\n| Workflow routing | labels `bug`, `feature`, `bootstrap`; workflows `tdd`, `feasibility`, `bootstrap`; fallback `implement` | Agent-system routing policy. Workflow identifiers are references, not compiled routing decisions. |\n| Filesystem/deployment | `.agents/skills`, `.archie/stages`, `.archie/gate.go`, `/data/worktree`, `/data/repo`, cache names/paths and prepared sentinel | Split into deployment settings, repository conventions, and genuine internal protocol markers. Paths an installation may relocate are settings. |\n| Packaged release data | `/usr/share/archie/CHANGELOG.md` | Packaging/deployment settings or paths derived from an installation prefix. |\n| Forge presentation | default label colour `bfd4f2` and state-label colours | Forge adapter presentation settings. |\n| Fallback system prompt | compiled fallback prompt | Agent-system default content, versioned and replaceable through its owning settings; safety constraints remain policy/code. |\n| Default paths and identities | config path, data paths, listen addresses, model roles and workflow names | Typed defaults belonging to their respective owners, never global constants consumed throughout the application. |\n\nThis inventory is intentionally broader than exported Go `const` declarations. A\nliteral hidden in a constructor, timeout, switch, retry loop, path join, prompt,\nlabel map, or client setup restricts behaviour just as effectively as an\nincorrectly used constant.\n\n## Constants that may remain\n\nA value may remain constant only when its owner documents why runtime variation\nwould be invalid. Typical valid cases are:\n\n- domain enum values and state names;\n- a versioned wire-protocol version or framing token;\n- reserved internal headers and callback prefixes that peers must interpret\n identically;\n- compile-time mathematical relationships;\n- sentinel errors and internal type identifiers.\n\nNames such as a stream, consumer, label, workflow, file, directory, network,\nmodel role, timeout, retry count, capacity, size limit, colour, prompt, command,\nor executable are not invariants merely because the current application has only\none of them.\n\nArchitecture tests SHOULD reject exported behavioural constants outside approved\ninvariant packages and SHOULD detect direct consumption of private default\nconstants outside their owning `DefaultSettings` construction.\n\n## Current mutation and reload ledger\n\nThe application currently changes behavioural values through unrelated paths:\n\n| Current change seam | Current behaviour | Required target |\n| ------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |\n| Telegram `/restart` in `cmd/archied` | Re-reads the global document and directly replaces Telegram token and allowed-user IDs; failure retains old values. | Submit a Telegram-settings candidate through the universal change protocol. Validate the complete candidate, promote atomically, audit the acting identity, and retain last-known-good. |\n| Active model selection | `chatModelManager.SetActiveModel` changes a model reference. | A conversation/session command changing runtime state. Validate against configured model instances and audit it; do not rewrite configuration. |\n| Active provider selection | `chatModelManager.SetActiveProvider` changes provider selection. | A conversation/session command changing runtime state, with the same distinction from provider-instance configuration. |\n| Persona selection | `PersonaRegistry.SetActive` mutates an in-memory per-session choice. | Agent-system session state. Persistence and audit requirements are owned by that domain, not the configuration loader. |\n| File and environment overlays | Applied while loading `internal/config`; defaults are filled by mutation in `finalize`. | Configuration-infrastructure input processing that produces separately typed candidates. Defaults and derivations remain visible in provenance. |\n| SIGHUP config reload (2026-08) | Re-resolves the file config, re-applies the database layer over it, validates, then atomically republishes through `config.Holder`. A failed reload, whether the file or the control plane caused it, keeps the running config and records `last_error`/`last_error_at` in `/api/config`. | Candidate promotion with last-known-good retention: the running snapshot is untouched on failure. A bounded observation period and actor audit are not yet implemented. |\n| Control-plane resource command (2026-09) | `ControlPlaneService.Command` validates a whole resource document against its owning feature, then writes it with an audit record (actor, source, request ID, expected version) under optimistic concurrency. Hosted on the State Store's gRPC server; the Web UI and messaging channels are its only clients. See `docs/prds/runtime-control-plane.md`. | Met for validate-stage-promote, actor audit and versioned history. Per-field health observation and validate-before-switch on live apply are not yet implemented. |\n\nAny further mutation seam discovered during migration MUST be added here before\nit is changed. Direct field assignment on a live settings object is prohibited.\n\n## Universal change protocol\n\nChanging configuration is an application command performed by an identity. It\nMUST:\n\n1. resolve the target owner and current version;\n2. parse into that owner's typed settings;\n3. validate syntax, semantic constraints, policy, references, and safe bounds;\n4. stage a candidate without altering the active value;\n5. run owner-defined readiness and health checks;\n6. atomically promote the candidate and record actor, source, reason, diff,\n prior version, and new version;\n7. observe the affected capability for a bounded period;\n8. retain the new version on success, or automatically restore last-known-good\n on failure;\n9. report the rejected or rolled-back candidate without terminating unaffected\n capabilities.\n\nStartup loading uses the same candidate rules. A bad user value therefore does\nnot partially mutate the application or make a previously healthy capability\nunavailable.\n\nSettings that cannot safely reload in-process are marked restart-required. The\nchange may still be validated and versioned immediately, but promotion occurs\nthrough a controlled restart with the same health and rollback semantics.\n\n## Implemented runtime reload and settings writes (2026-09)\n\nTwo mechanisms change behavioural values at runtime: SIGHUP re-reads the file\nconfig, and the control plane owns every database-backed setting. The dashboard\nruntime overlay that used to sit between them was deleted once the control plane\ncovered its surface; `-config-overlay` names a file overlay and is unrelated.\n\n### Published snapshot: `config.Holder`\n\nThe running configuration is a published snapshot held in `config.Holder`\n(`internal/config/holder.go`). Get returns a value under RLock; Set replaces the\nwhole snapshot. A published Config is immutable: reload constructs a fresh one\nand calls Set. `config.Clone` deep-copies every reference-type field so a failed\ndecode cannot mutate the published snapshot's shared maps. `Holder.Get` panics\non a nil receiver — a forgotten `NewHolder` must fail loudly, not boot a daemon\nthat silently runs on an empty config.\n\n### The reloadable-field criterion\n\nThe diff-and-warn on reload asks whether a changed field takes effect without a\nrestart. The criterion is: **a field is reloadable when every consumer re-reads\nit.** That question read as \"what does the daemon re-read\" while the daemon owned\nthe only Holder; it stopped being sufficient when webui began sharing that\nHolder, because webui handlers re-read on every request. The allowlist lives in\n`cmd/archied/reload.go` and is deliberately a deny-by-default list: a field added\nto Config later defaults to requires-restart, forcing whoever adds it to decide\ndeliberately. A reload that changes any non-allowlisted field logs\n\"requires a restart\" with the field names; a fully-applied reload logs Info.\n\n### SIGHUP reload\n\nSIGHUP re-resolves the file config, re-applies the database layer over it\n(`boot.runtimeConfig`, the same call boot makes), validates, and atomically\nrepublishes through the shared Holder. A failed reload keeps the running config\nand records `last_error`/`last_error_at` in `/api/config` so the stale state is\nvisible where the operator is looking. The three poll tickers re-read\n`PollInterval` per iteration and reset on change, so a reloaded interval\ngenuinely takes effect.\n\n### Settings writes go through the control plane\n\nDatabase-backed settings are read and written through\n`ControlPlaneService` on the State Store's gRPC server: catalog, query,\ncommand, watch. Each feature owns its resource's validation and commands;\nstorage writes versions and audit records only. `docs/prds/runtime-control-plane.md`\nis the specification.\n\nTwo properties matter to anyone changing this area:\n\n- **Bootstrap stays file-owned.** The daemon reads the file config first and\n layers database resources over it at boot (`boot.loadRuntimeConfig`). Values\n the daemon needs before it can reach the State Store — the database path,\n listen addresses, credentials — stay file-only by design.\n- **Reload re-reads them.** SIGHUP resolves the file document and then runs the\n same `boot.runtimeConfig` layering boot does, so a reload cannot revert a\n database-owned setting to its file value. A control plane that cannot answer\n fails the reload rather than publishing the file's values over the running\n ones. Workflow execution budgets arrive on a watch rather than in a resource\n query, so boot records the last published settings and the reload re-applies\n them from there.\n\n## Completion criteria\n\nThe configuration rearchitecture is complete only when:\n\n- `internal/config` no longer exists;\n- every behavioural value has one classification and owner;\n- every current field and hard-coded value in the ledgers has a recorded\n disposition;\n- domains and plugins receive only their typed settings;\n- all changes use semantic commands and the universal change protocol;\n- active, candidate, rejected, rolled-back, and last-known-good versions are\n queryable with secret-safe provenance;\n- generated documentation lists every external key, default, constraint,\n mutability class, owner, and deprecation;\n- architecture tests prevent a new central runtime configuration model, direct\n live-setting mutation, and incorrectly fixed behavioural values.\n\n## Startup policy: missing credential degrades, invalid config is fatal\n\nDecided 2026-07-30. Recovered 2026-08-09 from the pre-migration issue tracker.\n\nA missing **credential** disables a capability. Only an **invalid config** stops\nthe daemon. \"Forge is a feature, not a requirement.\"\n\nBefore `518a1a0`, an unresolvable forge token did `return 1` in\n`cmd/archied/main.go`, so a chat-only user who never configured GitHub got no\nchat, no gateway, nothing — and under a systemd unit with `Restart=on-failure`\nand no start limit it crash-looped every five seconds with the error scrolling\npast unread. Forge construction now lives in `resolveForge`, shared by the\nprimary forge and every configured identity, and degrades to `forge.NewNoop`\nwith a warning. The identity path had the same defect and was worse: one\nidentity's missing credential killed all the others.\n\nMalformed configuration still fails fast in `configuration.Validate`, well\nbefore this point. That distinction is the whole design.\n\nApply the same reasoning to any future capability credential, LLM keys\nespecially: start, report the capability as unavailable, let the operator see it.\nGuarded by `TestResolveForgeDegradesInsteadOfFailing`.\n\n**Known consequence, tracked separately:** `worktree.Manager` still receives the\nempty token, so pushes fail late rather than at startup.\n\n## Config generation belongs to `archied setup`, not `install.sh`\n\nDecided 2026-07-30 ([#368](https://github.com/samcharles93/archie-core/issues/368)).\nRecovered 2026-08-09 from the pre-migration issue tracker.\n\n`install.sh` must stop generating `config.toml`; `archied setup` owns it. The\ninstaller hand-rolling TOML has already drifted from the schema once: the\ngenerated-config branch hardcoded `key = ARCHIE_GITHUB_TOKEN` and\n`host = ${GITEA_URL}` regardless of the answers given, so choosing Gitea produced\na config naming a token the installer never wrote (archied then exited 1), and\nchoosing GitHub pointed `host` at `gitea.example.com`. `61a5b35` patched this\nwith a `forge_block` shell helper shared by both config paths — that helper is a\n**workaround**, deleted by\n[#72](https://github.com/samcharles93/archie-core/issues/72). The durable fix is\nthat the code writing the config is the code that reads it.\n\nWhy Go and not Python: the installer must run where nothing is installed yet, and\n`python3` with `tomllib` means 3.11+, which is not safe on LTS distros. `archied`\nis already being built two steps earlier.\n\n**Key constraint for the port:** `BurntSushi/toml` does **not** survive a\ndecode/encode round trip with comments intact, and `config.example.toml` carries\ndocumentation users are meant to hand-edit. Decide the write strategy\n([#412](https://github.com/samcharles93/archie-core/issues/412)) before writing\nany prompting code. Do not port a separate `config.yaml`/`auth.yaml` secret\nsplit — Archie already has `secret.SecretRef{engine,key}`, so setup writes the\nreference to TOML and the value to the secret engine.\n" + "body": "# Configuration\n\n**Status:** Approved; existing-type assignment is ongoing \n**Date:** 2026-07-28 \n**Tracking issue:** [#73](https://github.com/samcharles93/archie-core/issues/73)\n\n## Ownership\n\nConfiguration has two separate responsibilities:\n\n1. `internal/infrastructure/configuration` loads files, environment variables,\n secret references, overlays, and compatibility representations.\n2. Each domain and plugin owns the typed runtime settings required for its\n behaviour.\n\n`internal/app` translates external input into those settings, validates\ncross-capability combinations, and supplies each component only its own\nsettings.\n\nThe following are prohibited:\n\n- passing complete application configuration into a domain;\n- making a domain depend on TOML, YAML, environment-variable names, secret\n references, or a configuration library;\n- defining a central runtime configuration type imported across the system;\n- placing plugin options in an unrelated global configuration model.\n\n## Private input document\n\n`internal/infrastructure/configuration` MAY use a complete decoded document\nwhile discovering files, decoding sections, and applying overlays. It is an\ninput-boundary DTO only.\n\nThe document:\n\n- remains private to the configuration infrastructure;\n- is composed from separately defined feature sections;\n- is immediately translated by application composition;\n- is never returned as the application's runtime model;\n- is never passed through runtime components.\n\nExternal DTOs and domain runtime settings are separate types.\n\nPlugins supply external configuration definitions through a narrow extension\ncontract. The core loader MUST NOT gain a field or recognized filename for every\nplugin.\n\nInvalid configuration input MUST NOT replace active valid settings or terminate\nunaffected capabilities. Configuration changes use the candidate validation,\npromotion, audit, and recovery requirements in\n[safe-change-and-recovery.md](/dev/architecture/safe-change-and-recovery/).\n\n## Dissolution of `internal/config`\n\n`internal/config` has no target location. It MUST be dissolved and MUST NOT\nsurvive as a shared package, configuration domain, or renamed global model.\n\nIts loading behaviour moves to `internal/infrastructure/configuration`. Existing\ntypes and methods are reassigned by behaviour. Repository execution policy,\ndispatch policy, retry policy, ecosystem defaults, tool policy, memory settings,\nprovider settings, and channel settings move to their\nrespective owners.\n\n## Repository settings\n\nThe existing `config.Repo` MUST NOT move intact into a repository domain or\nshared settings package. A managed repository is an application association:\n\n- work intake owns which repository references an identity monitors;\n- task execution owns gates, preflight, test protection, ecosystem behaviour,\n and change limits;\n- task lifecycle owns retry behaviour;\n- scheduling owns concurrency policy;\n- workspace management owns base branch and persistent-storage requirements.\n\nApplication composition correlates these settings by repository reference. No\ndomain receives a universal repository profile.\n\n## Documentation\n\nGenerated configuration reference describes external sections and their owning\ncapabilities. It MUST NOT establish a global Go configuration structure as the\npublic contract.\n\n## Value classification rule\n\nEvery value that controls behaviour MUST have an explicit owner and MUST be\nclassified as exactly one of:\n\n1. **Invariant** — changing it changes the meaning of the program, a wire\n protocol, or a domain concept. It may be a Go constant.\n2. **Runtime setting** — an operator, identity, domain, plugin, or deployment\n may legitimately choose a different value. It is typed configuration owned by\n the affected capability.\n3. **Policy value** — it expresses what is allowed, limited, routed, retried,\n protected, or gated. Its typed definition belongs to the owning domain and it\n is evaluated through the policy system.\n4. **Derived value** — it is computed from other settings and is not\n independently mutable.\n5. **Runtime state** — it changes as the application operates, such as the\n active model for a conversation. It is changed by a domain command and is not\n configuration merely because it can be changed.\n\nAn unclassified behavioural literal is a defect.\n\n`const` MUST NOT be used merely because a value currently has one default.\nConstants are reserved for actual invariants: enum discriminants, protocol\nversions, fixed compatibility tokens, and values whose alteration would change\nthe program's semantics rather than configure an installation.\n\nIn Go, a constant is a compile-time value and does not represent mutable runtime\nstorage. It generally has no address that can be observed or changed.\nArchitecturally, a constant therefore asserts that the value cannot change for\nthe lifetime of the application. Any value that may legitimately change while\nthe application is running, between application instances, or between\ndeployments is not a constant.\n\nA runtime setting may have a default, but the default is not its only access\npath. The owner exposes a typed settings structure, validation, and a\n`DefaultSettings` constructor or equivalent. Private constants used to implement\ndefaults do not turn those defaults into invariants and MUST NOT be consumed\ndirectly by runtime code.\n\nEach runtime setting and policy value MUST declare:\n\n- its owner;\n- type, units, validation constraints, and safe bounds;\n- default and derivation, if any;\n- whether it is startup-only, live-reloadable, or restart-required;\n- whether it may differ by installation, identity, repository, workflow,\n channel, or plugin instance;\n- whether it is secret, sensitive, or safe to report;\n- its current source, effective value, version, and last-known-good version;\n- the command and identity required to change it.\n\n## Current configuration migration ledger\n\nThis ledger records the current meanings hidden inside `internal/config`. It is\nthe source inventory for dissolving that package; rows are split where the\nexisting structure combines different owners.\n\n| Current values | Current location | Target owner and requirement |\n| ----------------------------------------------------------------------------------------------- | --------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- |\n| `WorkDir`, `DBPath` | root `Config` | Workspace/storage infrastructure settings. `DBPath` is the prefix for independent task and conversation SQLite files. Defaults derive from the platform data directory. |\n| `SkillsDir`, `PluginDir` | root `Config` | Skill and plugin infrastructure settings, independently typed. |\n| `PollInterval` | root `Config` | Work-intake scheduling setting; not a daemon-wide constant. |\n| `Label`, `BotUser`, `BotEmail` | root `Config` | Identity and dispatch associations. `BotEmail` may be derived from the selected forge account but remains overrideable. |\n| `MaxRetries` | root `Config`, `Repo` | Retry policy owned by the operation being retried. The duplicate global/repository representation must disappear. |\n| `DiffCapLines` | root `Config` | Agent-change policy value. |\n| `MaxSteps`, `WallClock`, `GateMaxFailures` | `Budgets` | Workflow-execution policy, with units and zero/unlimited semantics made explicit. |\n| `Class`, `APIKeyEnv`, `BaseURL` | `Provider` | Provider instance settings. Secret lookup becomes a `SecretRef`, not an environment variable name leaking into the provider domain. |\n| `Mode`, `Command`, `Env` | `LegacyAgent` | Decode-only compatibility input for the removed executor selector. Delete after the config migration window; it has no runtime settings owner. |\n| `Type`, `Host`, `Token` / `TokenEnv` (older form) | `Forge` | Forge adapter settings. The older `TokenEnv` input is accepted only at the loader boundary. |\n| `Trigger`, `AckReaction`, label names | `Dispatch` | Work-intake and forge presentation policy. Label vocabulary is configurable per forge/identity association. |\n| `Engine` | `MemoryConfig` | Memory engine selection. `internal/domain/memory` is the only engine family; per-engine settings remain owned and validated by that engine. |\n| `Name`, `Transport`, `Command`, `Args`, `WorkDir`, `URL` | `MCPServer` | One typed MCP server instance specification. Transport-specific variants must reject fields belonging to other variants. |\n| `MaxResultChars` | `ToolPolicy` | Tool-execution policy values, not generic daemon configuration. |\n| `URL`, `TokenEnv` | `NATSConfig` | NATS transport settings with a secret reference replacing the environment-name field. |\n| `Image`, `MaxConcurrency`, `MaxUptime`, `VolumeTTL`, `PullPolicy`, `Network` | `ContainerConfig` | Mandatory managed-worker infrastructure settings and resource policy. The legacy `enabled` input is decode-only. |\n| model mappings, email, webhook, Telegram | `ChatConfig` | Separate channel instance settings plus channel-routing associations; there is no chat-wide settings owner. |\n| `AllowedUserIDs`, `Token` / `TokenEnv` (older form), update commands | Telegram config | Telegram adapter settings and access policy. `Token` is the preferred `secret.SecretRef`; `TokenEnv` remains a backwards-compatible fallback. |\n| listen/relay addresses | email config | Email adapter instance settings. |\n| webhook target | `Notify` | Notification adapter instance settings. |\n| web listen address | `Web` | Web transport settings. |\n| `listen` under `[services.state]` and `[services.gateway]` | `ServiceConnection` | The bind address of the process that owns the service. Read only by that process, and defaulted to the address its clients dial so the pair cannot drift. |\n| forge, dispatch, models, providers, containers, memory, tools and repositories | `IdentityConfig` | Existing composite duplicates unrelated settings. Replace it with associations from an `IdentityID` to independently owned configured instances. |\n| owner, name, base, gate, protection, ecosystem, preflight, storage, retry and concurrency | `Repo` | Split across work intake, workspace, workflow execution, policy, gate, and scheduling as described in Repository settings. |\n| `TaskConfig`, `TaskForge` | worker snapshot | Replace with the minimum immutable, versioned WorkflowExecution input assembled by the application. It is not a second configuration model. |\n| `Extra` and arbitrary maps | root/provider config | Allowed only at an extension boundary whose owning plugin/provider supplies schema and validation. Core configuration never consumes an untyped catch-all. |\n\n## Behavioural literals and incorrectly fixed values\n\nThe following current values affect useful behaviour but are not represented as\nowned typed settings or policy. They MUST be classified and migrated before the\nold configuration package is removed. The listed target is the architectural\ndecision; exact setting names are implementation details.\n\n| Capability | Values fixed in current code | Required classification |\n| ---------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |\n| Chat agent | maximum steps `8` in `cmd/archied` | Workflow/chat execution policy. |\n| Context compression | enabled, threshold `0.5`, context size `128000`, protect first `3` and last `20`, compression marker, token estimate of four characters per token | Model/context runtime settings. The marker is presentation; the estimate is provider/model strategy, not a universal truth. |\n| Tool guardrails | escalation thresholds `3`, `5`, `10`, `2` | Tool-use policy with named meanings and bounded defaults. |\n| Tool output | per-result cap `50000` characters, no aggregate turn cap, preview `500` characters | Tool-execution resource and presentation settings. Per-result spill/truncation remains; turn continuation is not character- or token-gated. |\n| Pairing/access | code TTL `1h`, maximum pending `3`, rate window `10m`, failures `5`, lockout `1h`, maximum lockout `24h` | Access policy. Code length/alphabet and salt length require an explicit security-constraint decision; they are not automatically valid constants. |\n| NATS daemon consumer | stream `ARCHIE_TASKS`, consumer `archie-daemon`, deduplication `2m`, poll `2s`, deliveries `3`, acknowledgement wait `5m`, inactive threshold `24h`, subjects | NATS topology settings plus delivery policy. Subject grammar or protocol headers may be invariants only if declared as versioned wire contracts. |\n| NATS worker consumer | stream `ARCHIE_TASKS`, consumer `archie-agent`, deduplication `2m`, poll `5s`, acknowledgement wait `30m`, deliveries `3`, inactive threshold `24h`, queue group and retry delay `1s` | Worker transport settings and delivery/retry policy. Shared stream defaults must have one owner rather than two copies. |\n| Worker readiness | readiness timeout `20s`, retry delay `250ms` | Worker lifecycle and retry policy. |\n| Execution/RPC | agent execution `30m`, task-run RPC `60s`, worktree RPC `15m` | Per-operation timeout policy. |\n| Plugin/tool cleanup | plugin rollback `10s`, tool-provider cleanup `10s`, MCP-provider cleanup `10s` | Lifecycle shutdown/rollback policy owned by each engine, with a shared policy vocabulary where appropriate. |\n| Memory read path | per-scope record limit `20`, rendered `\u003cmemory\u003e` block byte cap `8192` | Memory read policy owned by the engine family (`internal/domain/memory`). |\n| Built-in memory store | per-file size limit `100KiB`, query result limit `100` | Split into memory resource policy, input policy, and presentation settings. |\n| Memory safety scanner | compiled prompt-injection and sensitive-data patterns with fixed warn/block consequences | Versioned memory-domain policy definitions. Patterns and consequences must be inspectable and replaceable without editing scanner code. |\n| MCP stdio | initial backoff `500ms`, maximum backoff `30s`, unlimited default retries, send and shutdown `5s` | MCP transport retry, request, and lifecycle policy. Unlimited retries must be explicit rather than a zero-value accident. |\n| MCP SSE/HTTP | reconnect `1s` to `30s`, POST/request timeout `30s` | MCP transport settings and retry policy. |\n| Message framing | MCP frame `4MiB`, worker protocol frame `8MiB`, diagnostic `64KiB`, media/base64 `64MiB` | Resource/security policy unless a published wire protocol mandates the limit. Limits must be configurable only within safe bounds. |\n| Webhooks and web UI | webhook bind `0.0.0.0:8644`, webhook shutdown `5s`, web read-header `5s`, web shutdown `2s` | Transport instance and lifecycle settings. |\n| Telegram | typing refresh `4s`, draft interval `700ms`, update-action TTL `10m` | Telegram adapter presentation and lifecycle settings. |\n| Telegram limits | message length `4000` and callback/update prefixes | Record the upstream platform constraint and version it as an adapter compatibility limit; prefixes may remain wire constants when changing them would invalidate outstanding callbacks. |\n| Containers | stop and pull timeouts `10s` and `30s` | Container lifecycle policy. |\n| HTTP clients | feasibility notification timeout `10s`; other paths use `http.DefaultClient` | Adapter request policy. Every outbound client requires an explicit timeout profile. |\n| Workflow routing | labels `bug`, `feature`, `bootstrap`; workflows `tdd`, `feasibility`, `bootstrap`; fallback `implement` | Agent-system routing policy. Workflow identifiers are references, not compiled routing decisions. |\n| Filesystem/deployment | `.agents/skills`, `.archie/stages`, `.archie/gate.go`, `/data/worktree`, `/data/repo`, cache names/paths and prepared sentinel | Split into deployment settings, repository conventions, and genuine internal protocol markers. Paths an installation may relocate are settings. |\n| Packaged release data | `/usr/share/archie/CHANGELOG.md` | Packaging/deployment settings or paths derived from an installation prefix. |\n| Forge presentation | default label colour `bfd4f2` and state-label colours | Forge adapter presentation settings. |\n| Fallback system prompt | compiled fallback prompt | Agent-system default content, versioned and replaceable through its owning settings; safety constraints remain policy/code. |\n| Default paths and identities | config path, data paths, listen addresses, model roles and workflow names | Typed defaults belonging to their respective owners, never global constants consumed throughout the application. |\n\nThis inventory is intentionally broader than exported Go `const` declarations. A\nliteral hidden in a constructor, timeout, switch, retry loop, path join, prompt,\nlabel map, or client setup restricts behaviour just as effectively as an\nincorrectly used constant.\n\n## Constants that may remain\n\nA value may remain constant only when its owner documents why runtime variation\nwould be invalid. Typical valid cases are:\n\n- domain enum values and state names;\n- a versioned wire-protocol version or framing token;\n- reserved internal headers and callback prefixes that peers must interpret\n identically;\n- compile-time mathematical relationships;\n- sentinel errors and internal type identifiers.\n\nNames such as a stream, consumer, label, workflow, file, directory, network,\nmodel role, timeout, retry count, capacity, size limit, colour, prompt, command,\nor executable are not invariants merely because the current application has only\none of them.\n\nArchitecture tests SHOULD reject exported behavioural constants outside approved\ninvariant packages and SHOULD detect direct consumption of private default\nconstants outside their owning `DefaultSettings` construction.\n\n## Current mutation and reload ledger\n\nThe application currently changes behavioural values through unrelated paths:\n\n| Current change seam | Current behaviour | Required target |\n| ------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |\n| Telegram `/restart` in `cmd/archied` | Re-reads the global document and directly replaces Telegram token and allowed-user IDs; failure retains old values. | Submit a Telegram-settings candidate through the universal change protocol. Validate the complete candidate, promote atomically, audit the acting identity, and retain last-known-good. |\n| Active model selection | `chatModelManager.SetActiveModel` changes a model reference. | A conversation/session command changing runtime state. Validate against configured model instances and audit it; do not rewrite configuration. |\n| Active provider selection | `chatModelManager.SetActiveProvider` changes provider selection. | A conversation/session command changing runtime state, with the same distinction from provider-instance configuration. |\n| Persona selection | `PersonaRegistry.SetActive` mutates an in-memory per-session choice. | Agent-system session state. Persistence and audit requirements are owned by that domain, not the configuration loader. |\n| File and environment overlays | Applied while loading `internal/config`; defaults are filled by mutation in `finalize`. | Configuration-infrastructure input processing that produces separately typed candidates. Defaults and derivations remain visible in provenance. |\n| SIGHUP config reload (2026-08) | Re-resolves the file config, re-applies the database layer over it, validates, then atomically republishes through `config.Holder`. A failed reload, whether the file or the control plane caused it, keeps the running config and records `last_error`/`last_error_at` in `/api/config`. | Candidate promotion with last-known-good retention: the running snapshot is untouched on failure. A bounded observation period and actor audit are not yet implemented. |\n| Control-plane resource command (2026-09) | `ControlPlaneService.Command` validates a whole resource document against its owning feature, then writes it with an audit record (actor, source, request ID, expected version) under optimistic concurrency. Hosted on the State Store's gRPC server; the Web UI and messaging channels are its only clients. See `docs/prds/runtime-control-plane.md`. | Met for validate-stage-promote, actor audit and versioned history. Per-field health observation and validate-before-switch on live apply are not yet implemented. |\n\nAny further mutation seam discovered during migration MUST be added here before\nit is changed. Direct field assignment on a live settings object is prohibited.\n\n## Universal change protocol\n\nChanging configuration is an application command performed by an identity. It\nMUST:\n\n1. resolve the target owner and current version;\n2. parse into that owner's typed settings;\n3. validate syntax, semantic constraints, policy, references, and safe bounds;\n4. stage a candidate without altering the active value;\n5. run owner-defined readiness and health checks;\n6. atomically promote the candidate and record actor, source, reason, diff,\n prior version, and new version;\n7. observe the affected capability for a bounded period;\n8. retain the new version on success, or automatically restore last-known-good\n on failure;\n9. report the rejected or rolled-back candidate without terminating unaffected\n capabilities.\n\nStartup loading uses the same candidate rules. A bad user value therefore does\nnot partially mutate the application or make a previously healthy capability\nunavailable.\n\nSettings that cannot safely reload in-process are marked restart-required. The\nchange may still be validated and versioned immediately, but promotion occurs\nthrough a controlled restart with the same health and rollback semantics.\n\n## Implemented runtime reload and settings writes (2026-09)\n\nTwo mechanisms change behavioural values at runtime: SIGHUP re-reads the file\nconfig, and the control plane owns every database-backed setting. The dashboard\nruntime overlay that used to sit between them was deleted once the control plane\ncovered its surface; `-config-overlay` names a file overlay and is unrelated.\n\n### Published snapshot: `config.Holder`\n\nThe running configuration is a published snapshot held in `config.Holder`\n(`internal/config/holder.go`). Get returns a value under RLock; Set replaces the\nwhole snapshot. A published Config is immutable: reload constructs a fresh one\nand calls Set. `config.Clone` deep-copies every reference-type field so a failed\ndecode cannot mutate the published snapshot's shared maps. `Holder.Get` panics\non a nil receiver — a forgotten `NewHolder` must fail loudly, not boot a daemon\nthat silently runs on an empty config.\n\n### The reloadable-field criterion\n\nThe diff-and-warn on reload asks whether a changed field takes effect without a\nrestart. The criterion is: **a field is reloadable when every consumer re-reads\nit.** That question read as \"what does the daemon re-read\" while the daemon owned\nthe only Holder; it stopped being sufficient when webui began sharing that\nHolder, because webui handlers re-read on every request. The allowlist lives in\n`cmd/archied/reload.go` and is deliberately a deny-by-default list: a field added\nto Config later defaults to requires-restart, forcing whoever adds it to decide\ndeliberately. A reload that changes any non-allowlisted field logs\n\"requires a restart\" with the field names; a fully-applied reload logs Info.\n\n### SIGHUP reload\n\nSIGHUP re-resolves the file config, re-applies the database layer over it\n(`boot.runtimeConfig`, the same call boot makes), validates, and atomically\nrepublishes through the shared Holder. A failed reload keeps the running config\nand records `last_error`/`last_error_at` in `/api/config` so the stale state is\nvisible where the operator is looking. The three poll tickers re-read\n`PollInterval` per iteration and reset on change, so a reloaded interval\ngenuinely takes effect.\n\n### Settings writes go through the control plane\n\nDatabase-backed settings are read and written through\n`ControlPlaneService` on the State Store's gRPC server: catalog, query,\ncommand, watch. Each feature owns its resource's validation and commands;\nstorage writes versions and audit records only. `docs/prds/runtime-control-plane.md`\nis the specification.\n\nTwo properties matter to anyone changing this area:\n\n- **Bootstrap stays file-owned.** The daemon reads the file config first and\n layers database resources over it at boot (`boot.loadRuntimeConfig`). Values\n the daemon needs before it can reach the State Store — the database path,\n listen addresses, credentials — stay file-only by design.\n- **Reload re-reads them.** SIGHUP resolves the file document and then runs the\n same `boot.runtimeConfig` layering boot does, so a reload cannot revert a\n database-owned setting to its file value. A control plane that cannot answer\n fails the reload rather than publishing the file's values over the running\n ones. Workflow execution budgets arrive on a watch rather than in a resource\n query, so boot records the last published settings and the reload re-applies\n them from there.\n\n## Completion criteria\n\nThe configuration rearchitecture is complete only when:\n\n- `internal/config` no longer exists;\n- every behavioural value has one classification and owner;\n- every current field and hard-coded value in the ledgers has a recorded\n disposition;\n- domains and plugins receive only their typed settings;\n- all changes use semantic commands and the universal change protocol;\n- active, candidate, rejected, rolled-back, and last-known-good versions are\n queryable with secret-safe provenance;\n- generated documentation lists every external key, default, constraint,\n mutability class, owner, and deprecation;\n- architecture tests prevent a new central runtime configuration model, direct\n live-setting mutation, and incorrectly fixed behavioural values.\n\n## Startup policy: missing credential degrades, invalid config is fatal\n\nDecided 2026-07-30. Recovered 2026-08-09 from the pre-migration issue tracker.\n\nA missing **credential** disables a capability. Only an **invalid config** stops\nthe daemon. \"Forge is a feature, not a requirement.\"\n\nBefore `518a1a0`, an unresolvable forge token did `return 1` in\n`cmd/archied/main.go`, so a chat-only user who never configured GitHub got no\nchat, no gateway, nothing — and under a systemd unit with `Restart=on-failure`\nand no start limit it crash-looped every five seconds with the error scrolling\npast unread. Forge construction now lives in `resolveForge`, shared by the\nprimary forge and every configured identity, and degrades to `forge.NewNoop`\nwith a warning. The identity path had the same defect and was worse: one\nidentity's missing credential killed all the others.\n\nMalformed configuration still fails fast in `configuration.Validate`, well\nbefore this point. That distinction is the whole design.\n\nApply the same reasoning to any future capability credential, LLM keys\nespecially: start, report the capability as unavailable, let the operator see it.\nGuarded by `TestResolveForgeDegradesInsteadOfFailing`.\n\n**Known consequence, tracked separately:** `worktree.Manager` still receives the\nempty token, so pushes fail late rather than at startup.\n\n## Config generation belongs to `archied setup`, not `install.sh`\n\nDecided 2026-07-30 ([#368](https://github.com/samcharles93/archie-core/issues/368)).\nRecovered 2026-08-09 from the pre-migration issue tracker.\n\n`install.sh` must stop generating `config.toml`; `archied setup` owns it. The\ninstaller hand-rolling TOML has already drifted from the schema once: the\ngenerated-config branch hardcoded `key = ARCHIE_GITHUB_TOKEN` and\n`host = ${GITEA_URL}` regardless of the answers given, so choosing Gitea produced\na config naming a token the installer never wrote (archied then exited 1), and\nchoosing GitHub pointed `host` at `gitea.example.com`. `61a5b35` patched this\nwith a `forge_block` shell helper shared by both config paths — that helper is a\n**workaround**, deleted by\n[#72](https://github.com/samcharles93/archie-core/issues/72). The durable fix is\nthat the code writing the config is the code that reads it.\n\nWhy Go and not Python: the installer must run where nothing is installed yet, and\n`python3` with `tomllib` means 3.11+, which is not safe on LTS distros. `archied`\nis already being built two steps earlier.\n\n**Key constraint for the port:** `BurntSushi/toml` does **not** survive a\ndecode/encode round trip with comments intact, and `config.example.toml` carries\ndocumentation users are meant to hand-edit. Decide the write strategy\n([#412](https://github.com/samcharles93/archie-core/issues/412)) before writing\nany prompting code. Do not port a separate `config.yaml`/`auth.yaml` secret\nsplit — Archie already has `secret.SecretRef{engine,key}`, so setup writes the\nreference to TOML and the value to the secret engine.\n" }, { "url": "/dev/architecture/dashboard-agent-tools/", @@ -64,7 +64,7 @@ "url": "/dev/architecture/migration-decisions/", "title": "Architecture Migration Decisions", "section": "Architecture", - "body": "# Architecture Migration Decisions\n\n**Status:** Active migration inventory \n**Date:** 2026-07-28 \n**Tracking issue:** [#73](https://github.com/samcharles93/archie-core/issues/73)\n\n## Purpose\n\nThis document records what remains to be decided before Archie's current\nimplementation can be migrated into the approved domain-oriented structure.\n\nThe product and core domain model are not being redesigned. Existing Archie\nbehaviour and the existing session implementation are the migration\nbaseline. Remaining decisions MUST be grounded in current packages, consumers,\ntests, persistence, and runtime wiring.\n\n## Confirmed domain packages\n\n```text\ninternal/domain/\n identity/ (approved, not yet materialised as a package)\n agent/ (approved, not yet materialised as a package)\n workflow/\n messaging/ (canonical Message/Conversation types only; gateway migration pending, see section 2)\n workintake/\n plugin/ (approved, not yet materialised as a package)\n curator/\n mapping/\n scheduling/\n```\n\n- `identity` owns persistent actors, lifecycle, ownership, attribution, and\n audit identity.\n- `agent` owns persistent assistants, specialisation, instructions,\n model-independent continuity, capability context, and associations with\n identity and memory.\n- `workflow` is a separate domain. It owns reusable, versioned Workflow\n definitions, WorkflowSteps, WorkflowExecutions, StepExecutions, Attempts,\n outcomes, and typed Workflow plugin contracts.\n- `messaging` owns canonical Messages, Conversations, branches, channel and\n source correlation, replies, acknowledgements, delivery attempts, events, and\n projections.\n- `workintake` owns admission, validation, deterministic routing, and accepted\n work requests for the optional transition into durable workflow-backed\n execution.\n- `plugin` owns generic plugin identity, discovery, compatibility, and lifecycle\n mechanics. It remains metadata-only; capability semantics and typed\n registration belong to the owning domain.\n- `curator` owns the curator engine family: long-running agent loops that\n reason over memory and perform maintenance tasks on a declared shape\n (check-in interval, tool set, attached memory engine). See epic\n archie-core-yp9.\n- `mapping` owns the payload field-mapping vocabulary: binding named fields to\n JSON paths inside a captured webhook payload, and detecting drift. See\n `docs/prds/payload-field-mapping.md`.\n- `scheduling` owns the ticker engine: deciding when a scheduled job runs and\n with what concurrency. It declares `JobSource`/`Runner` contracts and owns\n no storage or delivery. See epic archie-core-1786637496320-249-be6d9a20.\n\n**Reconciliation (2026-08-31):** `curator`, `mapping`, and `scheduling` existed\nin the tree without being added to this list when their epics landed. They are\nconfirmed as legitimate, narrowly-scoped domains under the same rules as the\noriginal six — each owns one capability family, declares typed contracts, and\nhas no cross-domain reach. This list was out of date, not the tree.\n\nConversely, `identity`, `agent`, `messaging`, and `plugin` are approved target\ndomains that do not yet exist as packages. Their behaviour currently lives in\npre-migration packages (see \"Complete package destination map\" below for the\nfull inventory). Do **not** create empty stub packages for them ahead of an\nactual migration PR — an unused stub is dead code and gives no test coverage.\nEach materialises when its corresponding migration decision below is resolved\nand implemented in the same change.\n\nMemory's final package placement remains undecided pending focused review of the\nexisting implementation.\n\nThe following are approved shared or composition packages, not domains:\n\n```text\ninternal/eventbus/\ninternal/policy/\ninternal/infrastructure/\ninternal/app/archied/\ninternal/app/agentworker/\n```\n\n## Remaining migration decisions\n\n### 1. Complete package destination map\n\nEvery current package MUST receive one explicit disposition:\n\n- move into a confirmed domain;\n- move under infrastructure as an implementation of a domain-owned contract;\n- move into application composition;\n- merge with an identified duplicate;\n- remain as an approved shared application contract;\n- move to a plugin, extra, deployment, example, or generated artifact;\n- be deleted after its replacement is proven.\n\nThe largest unreviewed areas include memory, tools and capabilities, workspace\nand worktree behaviour, gates, storage, events, web UI, containers, RPC\npackages, release management, and scheduling.\n\nThe destination matrix MUST name the behaviour owner, target path, dependencies\nto remove, state and contracts to preserve, migration prerequisites, and\ndeletion criteria.\n\n**`internal/webui` / UI Service — RATIFIED Phase 3 disposition (2026-09-07).**\nThe current owner is the in-process HTTP/SSE dashboard: SPA hosting, operator\nauthentication, UI response projection, configuration reads and mutations,\ntask/control APIs, and direct adapters to State Store, channel, event/log,\nmemory, curator, workflow, forge, and Gateway runtime values. The target owner\nis a standalone UI application process; `internal/webui` remains its HTTP\ntransport adapter, while Gateway, State Store, configuration, Work Intake,\nMessaging, and other capability owners expose the contracts behind those\nroutes. The migration preserves retained route schemas, auth/CSRF behavior,\ntask/capture/mapping/binding/session continuity, SSE replay semantics, and\ncommitted generated SPA assets. Prerequisites are the ratified UI boundary,\nroute-owner inventory, narrow remote adapters, configuration/auth/readiness\ncontracts, and process/conformance tests. Remove the shared live\n`config.Holder`, direct event/log bus, daemon stopper, forge client, channel\nmanager, capability registries, composite store access, and in-process UI\nlistener only after the focused criteria in\n[`docs/prds/ui-service-boundary.md`](/dev/prds/ui-service-boundary/) pass in\nproduction composition. The old construction and listener are deleted in the\nsame cutover change; no second UI authority remains.\n\n### 2. Session and Messaging migration\n\nThe existing session implementation is the migration baseline. The migration must\ndecide:\n\n- which session and message components are retained or adapted;\n- how Agent, user, channel, Conversation, and branch ownership are added;\n- how Archie's parallel gateway session implementation is replaced;\n- how canonical Messages become immutable persisted records;\n- how existing sessions and messages are migrated;\n- how outbound delivery, retry, acknowledgement, deduplication, and failure\n semantics are represented;\n- how current Telegram, email, webhook, forge, and Jira integrations migrate to\n Messaging infrastructure adapters.\n\nThe following are already decided and MUST NOT be reopened without contradictory\nimplementation evidence:\n\n- the composite Conversation identity;\n- immutable MessageIDs;\n- typed tool-call and tool-result messages;\n- exact branch lineage through ParentConversationID and ForkMessageID;\n- parent and child transcript isolation;\n- normal scoped-memory behaviour regardless of the originating Conversation;\n- Messaging and Work Intake ownership.\n\n**Decision (2026-08-31), resolving the open questions above:**\n\n`internal/gateway/` is the \"parallel gateway session implementation\" this\nsection names. It is retained as-is for now — a big-bang rewrite of live\nsession/compression/approval/branch logic across Telegram, web UI, email, and\nwebhook channels in one change is exactly the \"convoluted to satisfy edge\ncases\" failure mode CLAUDE.md's Revert-on-Flail rule warns about, and would\nrisk silent behavioural regressions in the one path every user-facing message\ntravels through.\n\nInstead, `internal/domain/messaging/` is materialised now with the pieces\nthat are already decided and don't require touching gateway: the canonical\n`Message` and `Conversation` types (composite Conversation identity, immutable\n`MessageID`, typed tool-call/tool-result message variants, branch lineage via\n`ParentConversationID`/`ForkMessageID`). These are pure domain types with no\ngateway dependency — a new package, not a rewrite of an existing one.\n\nMigrating `internal/gateway/`'s session, compression, approval, and branch\nlogic onto these types is real, cross-channel-risk work and is **not** done in\nthis pass. It is tracked as separate, appropriately-scoped bd issues (see\n`bd list --label messaging-migration`), sequenced channel-by-channel so each\nlands with its own behavioural verification instead of one large diff no one\ncan safely review.\n\n**Amended 2026-09-15.** `internal/infrastructure/gatewayrpc.StoreClient` is the\nadapter this migration will need: it implements `messaging.SessionStore` over\nthe chat service's session-store RPCs and is proven by\n`TestStoreClientPreservesCanonicalRecords`. It is deliberately unwired.\nProduction `archied` dials the standalone gateway for `ChatContract` only\n(`composeChatContract` → `gatewayrpc.Dial` returns a `*Client`, not a\n`*StoreClient`), and the daemon's session store is in-process SQLite via\n`makeTelegramSessionStore`. The test comment that says it mirrors production\ndescribes the target, not today's tree. Kept rather than deleted because the\nmigration it serves is deferred by this very section, and re-deriving a tested\nwire adapter once that lands is pure cost.\n\n**Phase 4 Messaging Service extraction (2026-09-19, `archie-core-8cda.6.1`).**\nThe process boundary for external channels (Telegram, email, webhook) is ratified\nin [`docs/prds/messaging-service-boundary.md`](/dev/prds/messaging-service-boundary/).\nMessaging is extracted into `cmd/archie-messaging` / `internal/app/archiemessaging`,\ncalling Gateway's `messaging.ChatContract` exclusively over gRPC. Channel frontends\nhold no direct database handles, daemon state, or model runtimes.\n\n**Telegram operator surface after extraction (2026-09-19, `archie-core-8cda.6.6`).**\n`setupTelegramGateway` wired eight `telegram.Gateway` seams the daemon could\nsatisfy from its own process. Each is resolved by where the fact it needs is\nactually owned, not by where it used to be built:\n\n| Seam | Resolution |\n|---|---|\n| `Version` | Sourced from the Gateway via `ChatSnapshot.Version`. The Messaging Service never stamps or infers a build version; `cmd/archie-messaging` deliberately takes no version ldflags, so it cannot report a partial upgrade as a matched one. |\n| `Updates` (`/update`) | Built in the Messaging Service from `[chat.telegram].update_check_command` / `update_install_command` — its own configuration. `Enrich` is dropped: `componentInstallTypeEnricher` reads `[nats]`, which this service must not decode, so the check command's own reported install type stands unmodified. |\n| `UpdateReportPath` | A local state file under `work_dir`, keyed by the same `sha256(bot_user)[:8]` scheme the daemon used, so a pending report is found rather than written afresh. |\n| `ReleaseAnnouncements` | **Deliberately left nil**, for the same reason as `RunningVersions`. `releaseannounce.Announcer` announces nothing unless each `Component` carries a parseable version, and the only versions this service can obtain are the Gateway's own build stamps, not archied's. Wiring the announcer without them yields a no-op that reads as configured, so it stays unwired until component self-reporting exists. |\n| `SetShowToolCalls` | Projected from `[chat].show_tool_calls`. |\n| `Reload` | Local: re-resolves the token and allowlist from this service's own config file and overlay. Reload was never a daemon fact. |\n| `RunningVersions` | **Deliberately left nil.** It exists to turn an installer's claim into a checked one, and only a component's own compiled-in build can vouch for it. The Messaging Service knows neither archied's build nor the observed archie-agent version (`daemon.AgentStatus`), and the Gateway's own `gatewayVersion` is a different binary's stamp — reporting it as the daemon's would manufacture exactly the false success the check exists to catch. Update reports therefore relay as unverified claims until a component self-report contract exists. |\n| `Dangerous` | Stays nil. No daemon composition ever set it; `/rollback` and `/stop` reported \"not configured\" before the extraction and still do. |\n\nThe Messaging Service consequently reads `work_dir`, `bot_user`, `[chat]` and\n`[health]` in addition to the PRD's `[chat.*]` / `[services.gateway]` list.\n`[health]` is the daemon's health URL, which `releaseupdate.CommandInstaller`\npolls from outside to confirm an update came back up; the rest of the PRD's\nprohibition list is untouched.\n\n**Phase 4 messaging severance: what must move, and the decision the PRD does\nnot cover (2026-09-22, `archie-core-1ng1`).** The PRD's deletion gate cannot\npass today. `cmd/archie-messaging/architecture_test.go` does not exist, and the\ncheck that section specifies -- `go list -deps ./cmd/archie-messaging` links\nzero banned runtime packages -- reports four:\n\n| banned package | PRD category |\n| --- | --- |\n| `modernc.org/sqlite` | State Store |\n| `internal/store` | State Store |\n| `internal/domain/workflow` | Workflow engine |\n| `internal/agentexec` | Workflow engine, via `internal/domain/workflow/agent.go` |\n\nAll four arrive through one import: `internal/app/archiemessaging/run.go`\nimports `internal/app/controlplane`, whose own dependency set is exactly those\nfour. The Messaging Service uses three symbols from that package\n(`NewRPCClient`, `RuntimeChatConfig`, `ChannelSettingsKind`); the package also\ncarries the store-backed `ResourceStore` server, the `Definition` registry and\nthe workflow step vocabulary. Sever the package. Amending the gate to tolerate\n`modernc.org/sqlite` and `internal/store` is rejected: a boundary you whitelist\naround is no longer a boundary.\n\n**What moves.**\n\n- Every kind constant. They are strings with no dependency, and the client names\n them.\n- The channel-settings vocabulary and its decode: `channelSettings`,\n `telegramSettings`, `webhookChannelSettings`, `emailSettings`,\n `rateLimitSettings`, `channelDuration`, and `decodeRenamingLegacyKeys`.\n- The generic client: `Client`, `NewRPCClient`, `RuntimeConfig` with\n `runtimeConfigFrom`, `runtimeChatConfigFrom`, the other `runtime*From`\n projections, and the `resourceReader` interface.\n- The persona watch (`WatchPersonas`, `AppliedPersonas`), which needs only\n `internal/domain/agent`.\n- `executionSettingsDocument`, the document alone.\n\n**What stays, and why.**\n\n- The store-backed server: `Server`, `NewServer`, `ResourceStore`,\n `ImportConfig`, `SeedSkip`, the `Definition` registry, and every other kind's\n seed, validator and `Normalize` hook.\n- `WorkflowDefinitionsClient` and `stepRegistry`, which decode through\n `internal/domain/workflow`.\n- `WatchWorkflowExecutionSettings` and `decodeSettings`: `decodeSettings`\n returns and validates `workflow.ExecutionSettings`, so any package holding it\n links the workflow domain and therefore `internal/agentexec`. This is why the\n client cannot move as one package -- see open question 2.\n- The schedules document, whose type is `cronstore.JobSpec` in\n `internal/infrastructure`. A domain vocabulary package may not import\n infrastructure.\n- The schema derivation, which describes the documents the server advertises.\n\n**Placement as landed (2026-09-22, `archie-core-1ng1`).** One new package,\n`internal/infrastructure/controlplanerpc`, holding both the vocabulary (kinds, the\nchannel-settings document and its decode) and the client\n(`NewRPCClient`, `Query`, `RuntimeChatConfig`, the exported `ResourceReader`, the\nwire sentinels and `ClientError`). A second `internal/domain/controlplane` package\nfor the vocabulary was planned and then measured out of the design: the vocabulary\nhas exactly one consumer, the client that projects it, and the store-backed side\nreads the same definitions through type aliases rather than a copy -- so a separate\ndomain package would have been a package with one consumer and a second name for\none thing. What made that safe is a measurement, not a preference:\n`internal/app/controlplane` was the ONLY direct import of the Messaging Service\nthat dragged a banned runtime (`gatewayrpc`, `staterpc`, `applystatus`, `messaging`,\n`configuration`, `readiness` and `health` were all clean), so severing that one\ndependency is sufficient, and the severance needed no third package. Promoting the\nvocabulary to a domain package stays open for the day a second consumer appears.\n\n**Settled by the implementation.** Two of the four open questions this decision\noriginally left are answered, in both cases by measuring rather than by preference:\n\n- The client did NOT become two packages. `WatchWorkflowExecutionSettings` and\n `WatchPersonas` stayed on `internal/app/controlplane`'s `Client`, which now\n *embeds* the new one, so the daemon's validated watch is still defined once and\n the read path still has one implementation -- which is what the second package\n was for -- with one fewer package to keep in step.\n- The schedules and workflow documents stayed put, and\n `internal/domain/workflow/task` is linked deliberately: it carries the Task\n status and view vocabulary, its own dependency set holds none of the banned\n runtimes, and `cmd/archie-messaging/architecture_test.go` carries it as an\n exception checked in both directions (the same entry, for the same reason, as\n `cmd/archie-ui/architecture_test.go`).\n\n**Still open.**\n\n1. Does the server also leave `internal/app/`? `organisation.md` reserves\n `internal/app/` for wiring and composition, and a store-backed gRPC server is\n infrastructure -- but the gate does not require the move, and it would multiply\n the diff across archied's wiring and the State Store binary. Tracked as\n `archie-core-8f4n`.\n2. Does `internal/domain/workflow`'s contract/engine split happen, and when? A\n contract type (`WorkflowDefinitionCollection`) drags `internal/agentexec` only\n because `agent.go` sits beside it. Independent of this severance, tracked as\n `archie-core-inh6`, and the shape `archie-core-8cda.7` will meet again.\n3. Does the vocabulary move to a domain package when a second consumer appears?\n\n**Both consumers keep compiling.** archied imports the new vocabulary for the\nkinds it names and the new client package for `RuntimeConfig` and the persona\nwatch, and keeps `internal/app/controlplane` for the server, the registry,\n`ErrVersionConflict` and the workflow-aware pieces. The State Store binary\ncontinues to call `NewServer` and `ResourceStore` unchanged. The Messaging\nService then imports only `internal/infrastructure/controlplanerpc`,\n`gatewayrpc` and `staterpc`, so `go list -deps ./cmd/archie-messaging` loses all\nfour banned packages and the gate can land green.\n\n**Channel state to the dashboard: one administrative surface, following the\nconfig-snapshot precedent (2026-09-22, `archie-core-8cda.6.8`).** The Messaging\nService now records its channels' lifecycle -- `internal/channels/status.Manager`,\none entry per composed channel, written by each channel's own report through\n`channels.Lifecycle` -- and nothing reads that across a process boundary.\n`/api/channels` answers with nothing because `webui.Server.Channels` has no\nproduction source, and POST `/api/channels/{id}/reload` answers 501 for the same\nreason. The carrier is the State Store, and the shape is the one\n`archie-core-ymut` already established for config snapshots\n(`docs/prds/state-store-contract.md`, \"the one surface whose writer is the daemon\nand whose reader is the UI process\"): a pair of administrative RPCs, deny-by-default\nunder `authorizesTaskScopedCall`, so a task-scoped grant reaches neither.\n\nWhat that decides, so the implementation need not invent it:\n\n- A channel's state is **runtime** state, not a settings document. It does not\n belong in the `ControlPlaneService` resource kinds, which are validated,\n versioned documents seeded from config; a channel that is failed is not a\n document anybody edits.\n- The reader is the UI process, which already dials the State Store and pumps its\n cursor, so the surface needs no new transport in either direction.\n- Adding the pair changes the contract's RPC count, so\n `docs/prds/state-store-contract.md`'s grouped table gains a group in the same\n change that adds the RPCs -- the table is the contract's own index, and a\n surface it does not list is a surface nobody finds.\n\n**Reload gets its own surface, with its own lifetime (decided 2026-09-22).** Reload\nruns the other way -- the UI asks the service to reload a channel -- and the two\ncandidates were a request surface the service observes and clears, or a field on\nthe status surface the service watches. It is the first. The reasoning, recorded\nbecause it would look arbitrary later:\n\n- A request queue and a status report do not share a lifetime, so they should not\n share a surface. A pending request is transient and has an outcome; a channel's\n state is durable and has a current value.\n- A \"please reload\" field on the status resource makes READING status capable of\n RETRIGGERING an action, and makes a transient request indistinguishable from\n state the moment anything polls it.\n- It would also invert the writer/reader relationship the precedent establishes:\n the service would have to clear a field on an object whose whole contract is\n that the service reports it.\n\nSo the status surface stays writer-is-service, and the request carrier is its own\nobject with its own states.\n\n`archie-core-8cda.6.9`'s component self-report is the **same shape**: a process\nreporting something about itself for the UI to read. Whether it shares this\nsurface or gets its own is a decision for `6.9`, and the one outcome to avoid is\ntwo homes for \"what a component reports about itself\".\n\n### 3. Identity data migration\n\nThe migration must define:\n\n- the durable identity schema and repository implementation;\n- bootstrap of the permanent system identity;\n- stable IdentityIDs for existing configured Agents;\n- migration from configured identity names, empty-string ownership, and bot\n usernames;\n- preservation of historical attribution;\n- identity-aware uniqueness and transport deduplication;\n- correction of container and RPC capability resolution so execution uses the\n owning identity's bindings;\n- removal of identity-string equality from access-control behaviour.\n\nExact schema and method signatures are implementation decisions. The approved\nIdentity lifecycle and attribution semantics are fixed migration constraints.\n\n### 4. Workflow migration\n\nThe current implementation must be transformed deliberately:\n\n```text\nstore.Task -\u003e WorkflowExecution\nmutable Stage -\u003e StepExecution history\nRetryCount / Attempt -\u003e Attempt records\nworkflow.Registry -\u003e versioned Workflow definitions\ncurrent stages -\u003e WorkflowStep definitions or Workflow plugins\n```\n\nThe migration must decide:\n\n- Workflow definition and version persistence;\n- the smallest accepted-work-request contract consumed from Work Intake;\n- the enforced WorkflowExecution state machine;\n- atomic state and domain-event persistence;\n- migration of existing task rows and transition history;\n- disposition of current skill, Yaegi, and stage-plugin implementations;\n- the versioned worker-handoff contract;\n- scheduler, claim, retry, crash-recovery, and waiting-human cutover.\n\nWorkflow remains a separate domain from Agent. A user or Agent may define or\ninvoke a Workflow, but Workflow semantics and plugins remain Workflow-owned.\n\n### 5. Memory placement and storage\n\nThe four required scopes are fixed:\n\n- global shared;\n- Agent-wide;\n- user-wide;\n- Agent-user relationship.\n\nA focused review of `internal/memory`, its providers, consumers, persistence,\nand tests had these decisions:\n\n- `internal/domain/memory` versus ownership within the Agent domain — settled\n (2026-09-02, below);\n- the authoritative memory record and revision model — settled by the PRD\n (2026-09-16, below);\n- provider and infrastructure boundaries — settled by the PRD (2026-09-16,\n below);\n- retrieval and access enforcement — settled by the PRD (2026-09-16, below);\n ranked retrieval remains open (below);\n- migration of existing memory data — open: existing data is not migrated;\n- provenance representation — settled by the PRD (2026-09-16, below).\n\nConversation branches do not create another memory scope. A memory action has\nthe same scoped effect regardless of which Conversation originated it.\n\n**`internal/domain/memory` ownership — DECIDED (2026-09-02).** The typed\nengine family lives at `internal/domain/memory`\n(archie-core-1786637490069-11-e245c170), mirroring `internal/domain/curator`'s\ncontract/registrar/registry split. Not owned within the Agent domain: memory\nis consumed by more than agent turns (curators, per archie-core-1786637499636)\nand the plugin engine rule requires its own family, not a capability bolted\nonto another domain's contract.\n\n**Authoritative record and revision model for the builtin engine — DECIDED\n(2026-09-02),** scoped to unblock archie-core-1786637499480 (ship the\nfile-writing store as the default engine). This decides only the builtin\nengine's mapping onto the new `Record{ID}`/`Forget(id)` contract, not the\nmemory model in general or a future non-file engine's model:\n\n- `internal/memory/builtin.Store` is unchanged. It has no concept of a\n stable per-block ID — blocks are addressed by content substring — and nothing\n in this decision requires changing that; existing `internal/memory` tests\n keep passing unmodified, per that issue's acceptance criteria.\n- A new infrastructure adapter (`internal/infrastructure/memory`, per\n `organisation.md`'s \"infrastructure implements domain contracts\" rule — not\n under `internal/domain/memory`, which stays contract-only) wraps one\n `builtin.Store` per identity.\n- **Identity → directory:** `hex(sha256(identity))`, not the identity string\n itself. Closes the path-traversal question without an allowlist regex, and\n sidesteps collisions from identities that differ only in characters a\n filesystem treats as equivalent.\n- **Record ID:** assigned once at `Write`, independent of content, so a later\n edit elsewhere in the file can never orphan a `Forget`. Realized as a\n leading marker line (`\u003c!--mem:\u003culid\u003e--\u003e`) on the block `builtin.Store.Add`\n writes; `Query`/`List` strip it back out of `Record.Content` before\n returning, and `Forget(id)` locates the block by that unique marker via the\n store's existing substring-based `Remove`. The marker is a pragmatic,\n reversible choice — plain-text records stay human-readable and\n human-editable, at the cost of one marker line an operator editing the file\n by hand should leave alone. Revisit if/when provenance representation\n (still open, below) settles on something richer.\n- **Kind → section:** `Observation.Kind` (default `\"general\"` when empty) is\n the section name, reusing `builtin.ValidateSectionName` — so an\n invalid `Kind` fails `Write` with the same validation error `memory_edit`\n already returns for an invalid section today, not a new error shape.\n\n**Memory engine: scopes, CRUD with retained revisions, and store placement —\nDECIDED (2026-09-16).** Supersedes the 2026-09-02 \"Authoritative record and\nrevision model for the builtin engine\" entry above, whose `\u003c!--mem:\u003culid\u003e--\u003e`\nlive marker cannot carry provenance. Authority:\n[`docs/prds/memory-engine-unification.md`](/dev/prds/memory-engine-unification/)\n(Approved); the contract is `internal/domain/memory/contract.go`. This register\nsupersedes; it does not erase.\n\n- **Four typed scopes, caller-named.** `internal/domain/memory` is addressed by\n `Scope{Kind, Agent, User}` over `global`, `agent`, `user` and `agent-user`,\n with `Validate` rejecting a component present where it is not required. A\n caller names the read and write sets (`Subject.Scopes()` /\n `WritableScopes()`; global is never model-writable) and the engine applies no\n policy of its own: **access control is naming a scope, and nothing else.** An\n engine that enforced policy would need to know about channels, sessions and\n bindings, and every backend would reimplement it. `Scope.Key()` length-prefixes\n each component, so two scopes can never share a key even when an opaque\n channel-native id embeds the separator.\n- **The caller resolves identity; the engine never does.** Resolution happens in\n the gateway turn path, where the native sender id and the session are both in\n hand, and is supplied per channel by the composition. A webhook's sender id is\n a route path, not a person, and treating it as one would give a URL an\n identity. An absent user id yields global and agent scopes only: \"show none of\n it\", never a wider fallback.\n- **CRUD supersedes, never overwrites.** `Store` is `Create`/`Get`/`Query`/\n `List`/`Update`/`Forget`/`Revisions` over the four scopes. `Update` appends the\n superseded state to the scope's `HISTORY.md` before replacing the live block;\n `Update.Expected` names the revision being replaced and returns\n `ErrStaleRevision` on mismatch, making a lost update between two processes\n sharing one scope file detectable. `Forget` records a deleted revision before\n removing the live block, so a record's provenance outlives its content; `Get`\n and `Forget` take a scope, because an id alone does not locate one.\n- **Provenance representation.** The live marker becomes JSON, since a\n `\u003c!--mem:\u003culid\u003e--\u003e` line cannot carry provenance, and `Record` retains the\n provenance `agent-system.md` requires — scope, author, originating user,\n source, and revision history. `Metadata` is dropped: it never round-tripped.\n- **Store and scanner placement.** The markdown store relocates to\n `internal/infrastructure/memory/builtin`, its correct home under\n `organisation.md`'s \"a package owns its on-disk format end-to-end\". The content\n scanner moves into `internal/domain/memory` and is applied in `Create` and\n `Update` — the single choke point every producer crosses, so curator-extracted\n content is covered too.\n\n**Open in this section.** Ranked retrieval is deferred: `Query.Text` stays in\nthe contract, unused by the prompt path. Existing memory data is not migrated:\n`\u003cworkDir\u003e/memory` and `\u003cworkDir\u003e/memory-engine` are left on disk, unreferenced.\nThe PRD leaves these undetermined and they remain open here: group-chat\nextraction, where the curator fails closed on mixed-sender sessions; two\nprocesses (`archied` and `archie-gateway`) holding one scope file, where\n`Update.Expected` detects but does not prevent a lost block; unbounded\n`HISTORY.md` with no compaction policy; whether the per-scope size bound is\nright; and whether the dashboard should carry a user identity.\n\n### 6. Runtime and process boundaries — DECIDED\n\n`archied` is the native resident orchestration and interactive-chat process.\nEvery autonomous repository workflow crosses one full-task core-NATS handoff\ninto a task-scoped `archie-agent` container. Embedded versus external NATS\nchanges broker deployment only; it never selects an executor. The former host\nin-process runner, per-stage NATS protocol, and subprocess runner are removed.\n\nForge, store, and worktree authority remains in `archied` and is exposed to the\ntask container through scoped RPC services. The worker runs the workflow and\nits worker-local ai-sdk loop; it does not receive forge credentials or direct\nstore access. See `docs/prds/embedded-nats.md`.\n\n**UI Service boundary — RATIFIED (2026-09-07).** Phase 3 has a focused\nauthority record in [`docs/prds/ui-service-boundary.md`](/dev/prds/ui-service-boundary/).\nThe UI Service owns HTTP/SPA delivery, UI DTOs, browser authentication, and\nits own endpoint/credential/readiness settings. It consumes Gateway and State\nStore contracts and does not receive `config.Holder`, daemon pointers, SQL or\nstore implementations, workflow registries, forge/channel/model runtimes, or\nresolved secrets. The current `internal/webui` implementation remains inside\n`archied` until the ratified migration gates prove route parity, adapter\nselection, failure/readiness behavior, and deletion of the shared-holder and\ndirect-access paths. The extraction cutover removes the in-process UI\nlistener in the same change; there is no dual-live UI authority.\n\n**Dashboard live event delivery — DECIDED (2026-09-09, `archie-core-za9f`).**\nThe extracted UI receives live activity by polling the State Store's existing\n`EventsSince` cursor, not by a new push-subscription RPC. No proto change, no\ncontract amendment beyond this record.\n\nThe UI process runs one event pump. It primes its watermark by paging\n`EventsSince` to the end at startup without delivering, then polls\n`EventsSince(watermark, limit)` on an interval and hands each new event to\n`webui.Server.Broadcast`, exactly where `internal/app/archied/main.go`'s\n`persistAndBroadcastEvents` hands events to it today. One pump serves every\nconnected browser, so cost is one indexed query per interval regardless of\nviewer count, and `Broadcast` is a no-op when nobody is watching.\n\nThree properties make the cursor sufficient, and a push hub unnecessary:\n\n- **The cursor cannot skip.** `internal/store/store.go` opens the database with\n `SetMaxOpenConns(1)`, so writes serialise and a row's `id` is assigned in\n commit order. There is no window in which a lower id becomes visible after a\n higher one, which is the failure mode that normally rules out polling an\n autoincrement cursor. SSE catch-up already depends on this same property.\n- **A broadcast is a wakeup, not the payload.** `sseStream.drain` treats each\n broadcast event as a signal to fill the persisted gap ahead of it via\n `catchUp`, then delivers the event only if catch-up did not already cover\n it, deduplicating on the `since` watermark. Substituting a poller for the\n in-process bus therefore changes no SSE semantics, and needs no change to\n `internal/webui/sse.go`. It also means a broadcast dropped to a stalled\n client self-heals on the next one.\n- **The table is the only complete fan-out point.** Events reach the store\n through several paths that never touch the daemon's bus, including the\n audit event `store.ArchiveTask` writes inside its transaction\n (`internal/store/store.go`) and the binding-dispatch failure\n `internal/daemon/daemon.go` inserts directly. A push hub would have to be\n hooked at every write site and at post-commit, and would silently miss any\n site added later that forgets to notify. Reading the table misses nothing.\n\nThe cost is one poll interval of latency on the activity feed, default 1s and\noperator-tunable. That is accepted: the feed reports task lifecycle history,\nand an operator action already gets its own synchronous HTTP response.\n\nThis resolves the `/events` row of the route-owner inventory in\n[`ui-service-boundary.md`](/dev/prds/ui-service-boundary/). `/api/logs` and\n`/api/logs/stream` are not resolved by it, they read the daemon's in-process\ndiagnostic feed and remain host-local (see `archie-core-8cda.5.4`).\n\nRevisit only if sub-second feed latency becomes a requirement, or if a second\nconsumer needs the same stream, at which point the fan-out belongs in the State\nStore process behind a `SubscribeEvents` RPC and this record is superseded.\n\n**Dashboard configuration page — DECIDED (2026-09-09, `archie-core-ymut`).**\nThe configuration page becomes read-only in the UI process for Phase 3. Reads\ncross the boundary as a published snapshot; writes are descoped, not\ncontracted. This amends the ratified boundary, which assumed a single admin\ncontract carrying both.\n\n**Reads.** `webui.ConfigView` is already the whole answer. It is a UI-owned,\nsecret-free value projection built in `handleConfig`: `ProviderView` carries\nthe API key's environment variable name and a `Configured` boolean, never a\nvalue, and `RepoView` is repository metadata only. It is already the exact\nshape the browser receives. So the daemon publishes that projection, plus\nprovenance, reload status and the overridden-key list, as a snapshot the UI\nreads; nothing new has to be designed, only moved. Two State Store RPCs carry\nit, a `PutConfigSnapshot` restricted to the administrative token and a\n`GetConfigSnapshot` the UI calls. The daemon republishes on boot and after\neach successful reload, which is precisely when the projection changes.\n\nThe State Store is the right holder because a published configuration snapshot\nis shared state, it is already dialled by the UI, and it costs no new listener,\nport, token or interceptor. The alternative, a config admin service on\n`archied`, would make the daemon a gRPC server for the first time. Today it is\npurely a client of Gateway and State Store; `grpc.NewServer` appears only in\n`RunGateway` and `RunStateStore`, which are the standalone binaries' entry\npoints, not the daemon's.\n\n**Writes.** `PATCH /api/config`, `POST /api/config/reset` and\n`PATCH /api/config/repos/{owner}/{name}` answer 503 in the UI process, and the\nSPA hides those controls rather than offering a button that fails. Editing is\n`config.toml` plus reload, which is already the only path for every setting\noutside the runtime-tunable allowlist.\n\nThis is a deliberate feature reduction and worth naming as one. The write path\nis not hard because of the transport; it is hard because the policy is\ndaemon-local and stays that way: `installUpdateConfigHandler` denies\nnon-runtime-tunable keys, applies a dotted overlay to a clone of the published\nconfig, validates the materialised result, writes overlay rows, and rebuilds\nprovenance. Contracting that means either moving the policy, which the\nboundary forbids, or exposing it verbatim as remote procedure calls, which\nbuilds a Configuration Service inside a migration bead. Neither is work this\nphase should be doing to keep one page's buttons alive.\n\nRevisit when dashboard configuration editing is picked up as a feature in its\nown right (`archie-core-1786637498420-327`, still open). At that point it gets\nan admin contract designed as a feature, and the Configuration Service this\ndocument already anticipates is its natural owner. Until then the UI renders\nconfiguration and does not write it, which is what the boundary always said\nabout ownership even when it assumed a wider contract.\n\nAmended 2026-09-11. This descope is a deferral, not a decision that the\ndashboard should be read-only. Name it that way in review: the daemon-side\npolicy was built and tested, then deleted by the cutover (`e037b95` removed\n`internal/app/archied/config_update.go` and its tests), so reviving it is\nrecoverable work, not a rewrite. `archie-core-j28m` carries the remaining\nquestion -- which process serves the policy and what transport carries the\nwrite -- and blocks the feature bead. The handlers, their error sentinels and\nthe `UpdateConfig`/`ResetConfig`/`UpdateRepoField` seams on `webui.Server` are\nkept deliberately for it; `ConfigView.Editable` follows whichever process holds\n`UpdateConfig`, so wiring that one field makes the page editable\n(`archie-core-ml30`).\n\nImplementation belongs to `archie-core-8cda.5.4`, the cutover that removes the\ndaemon's own dashboard. Nothing here is built yet.\n\n**Superseded 2026-09-21 by the runtime control plane.** The open question this\ndescope left -- which process serves the policy and what transport carries the\nwrite -- was answered by `docs/prds/runtime-control-plane.md`: neither a daemon\nadmin service nor a revival of the daemon-local policy. `ControlPlaneService` on\nthe State Store's existing gRPC server owns every database-backed setting, and\neach feature validates its own resource, so there is no daemon-local policy left\nto contract. `archie-core-j28m` is closed.\n\nConsequently the three write seams described above no longer exist.\n`PATCH /api/config`, `POST /api/config/reset` and\n`PATCH /api/config/repos/{owner}/{name}`, the `UpdateConfig`/`ResetConfig`/\n`UpdateRepoField` fields on `webui.Server`, and `ConfigView.Editable` were\ndeleted in `31622ec5` along with the runtime overlay store they wrote to.\n`GET /api/config` remains a read of the published projection, as this decision\nalways intended.\n\n### 7. Shared mechanics\n\nThe following cross-domain mechanics require exact contracts before dependent\npackages migrate:\n\n- event-bus delivery and failure semantics;\n- policy evaluation API, composition, and precedence;\n- configuration candidate preparation, health validation, atomic promotion,\n observation, and rollback;\n- service-specific health, bounded remediation, isolation, and escalation;\n- changelog association and grouping.\n\nThese shared mechanisms do not acquire ownership of domain meaning. Domains\ncontinue to own their commands, events, policies, settings, and consequences.\n\nThe Go documentation generator, its output placement, and drift checking are\ndefined in `generated-documentation.md`. The VitePress site and its Pages\ndeployment were removed on 2026-09-12 as an unnecessary build step; `docs/` is\nrepository documentation only. The rendering and publishing surface is an open\ndecision, and so are the renderer's page templates and any developer commands\nor CI sequence that would accompany one. Adapting the generator and implementing\nArchie's normalized documentation model remain migration work rather than open\nproduct architecture.\n\n**Amended 2026-09-15.** `internal/infrastructure/servicediscovery/nats`\nimplements the `servicediscovery.ServiceRegistry` contract (Connect, Register,\nResolve, Watch) over NATS KV and is fully tested, but no production code\nconstructs it and no config surface exposes it. This is a deliberate deferral,\nnot an oversight: service-to-service addressing belongs to the runtime and\nprocess boundary work in section 6, which currently pins container addressing\nby injecting the daemon's startup NATS URL rather than resolving through a\nregistry. The scaffold is kept for that work; re-derive it only if the\naddressing model changes.\n\n### 8. Cutover sequence\n\nThe final migration plan must order:\n\n- compatibility adapters;\n- schema migrations and data backfills;\n- any required dual-read or transitional paths;\n- domain-by-domain package replacement;\n- production wiring changes;\n- legacy package removal;\n- architecture tests preventing dependency regression;\n- focused, integration, race, and full quality gates.\n\nEach legacy package requires objective removal criteria. No compatibility path\nmay become an indefinite second implementation.\n\n\n## Completion criteria\n\nMigration design is complete when:\n\n- every current package has one approved owner and destination;\n- every mutable record has one authoritative owner and migration path;\n- all external and cross-process contracts have versioned owners;\n- existing behaviour and data have explicit parity requirements;\n- process boundaries have evidence-backed justification;\n- the cutover order preserves a working application;\n- legacy deletion criteria and deterministic architecture tests are defined;\n- no unresolved decision requires an implementer to invent domain semantics.\n\nExact Go APIs, SQL statements, and mechanical refactoring steps MAY be finalized\nduring implementation when they do not change these semantics.\n\n## Recorded migration position (2026-07-29)\n\nRecovered 2026-08-09 from the pre-migration issue tracker. This section records\ndecisions the maintainer approved that the repository did not otherwise capture.\n\n**Completed:** `internal/infrastructure/eventbus/nats` (rebuilt, not relocated);\n`internal/infrastructure/configuration` (loading moved out of `internal/config`,\nwhich is now types-only and does no I/O); `internal/eventbus` (broker-neutral\ncontract); `internal/domain/workintake` (the first `internal/domain/` package,\nowning `TaskEnvelope`, the routing `Kind`, the label vocabulary, and task\nsubjects).\n\n**DECISION — private input DTO deferred (approved).**\n`configuration.Document.Config` is still `internal/config.Config` rather than the\nprivate per-domain input DTO this document mandates. Building an ~800-line\nparallel type tree while every consumer still reads `config.Config` would mean\ntwo shapes kept in sync by hand, with no consumer for the new one. It is named as\na field so the compromise is visible at each use site, with the deletion\ncondition recorded in that package's `doc.go`. Do this **with** the\n`internal/config` dissolution, not ahead of it.\n\n**Agreed next: `internal/app/{archied,agentworker}`.** `run()` in\n`cmd/archied/main.go` is ~460 lines, against `organisation.md`'s rule that\n`cmd/*` must not contain substantive wiring or act as a service locator. It is\nalso the last non-infrastructure holder of the NATS SDK, and it is where the\nconfig DTO translation will land, so it unblocks the dissolution above.\n\n**Method constraint.** Do not write a separate recorded package-review document\nper area when the decisions are already dictated by\n`dependencies-and-contracts.md`. That is the ceremony pattern this project\ndeliberately avoids.\n\n## OPEN — `[chat.*]` config section naming\n\n`configuration.md:160` already states the position: `ChatConfig` holds \"separate\nchannel instance settings... there is no chat-wide settings owner\". The config\nsection name has not followed, so a settings block named for a product surface\nowns what the architecture calls channel state.\n\nTwo candidates, neither settled: `[channel.*]` and `[messaging.channel.*]`. The\nsecond reads as a domain path, and no config section in this repo is currently\nnamed after a domain package, so adopting it sets a precedent for every other\nsection rather than fixing one name.\n\nBlocked on a maintainer decision, not on evidence. This is a rename with an\noperator-visible TOML break, so it should land with the `internal/config`\ndissolution recorded above rather than on its own.\n" + "body": "# Architecture Migration Decisions\n\n**Status:** Active migration inventory \n**Date:** 2026-07-28 \n**Tracking issue:** [#73](https://github.com/samcharles93/archie-core/issues/73)\n\n## Purpose\n\nThis document records what remains to be decided before Archie's current\nimplementation can be migrated into the approved domain-oriented structure.\n\nThe product and core domain model are not being redesigned. Existing Archie\nbehaviour and the existing session implementation are the migration\nbaseline. Remaining decisions MUST be grounded in current packages, consumers,\ntests, persistence, and runtime wiring.\n\n## Confirmed domain packages\n\n```text\ninternal/domain/\n identity/ (approved, not yet materialised as a package)\n agent/ (approved, not yet materialised as a package)\n workflow/\n messaging/ (canonical Message/Conversation types only; gateway migration pending, see section 2)\n workintake/\n plugin/ (approved, not yet materialised as a package)\n curator/\n mapping/\n scheduling/\n```\n\n- `identity` owns persistent actors, lifecycle, ownership, attribution, and\n audit identity.\n- `agent` owns persistent assistants, specialisation, instructions,\n model-independent continuity, capability context, and associations with\n identity and memory.\n- `workflow` is a separate domain. It owns reusable, versioned Workflow\n definitions, WorkflowSteps, WorkflowExecutions, StepExecutions, Attempts,\n outcomes, and typed Workflow plugin contracts.\n- `messaging` owns canonical Messages, Conversations, branches, channel and\n source correlation, replies, acknowledgements, delivery attempts, events, and\n projections.\n- `workintake` owns admission, validation, deterministic routing, and accepted\n work requests for the optional transition into durable workflow-backed\n execution.\n- `plugin` owns generic plugin identity, discovery, compatibility, and lifecycle\n mechanics. It remains metadata-only; capability semantics and typed\n registration belong to the owning domain.\n- `curator` owns the curator engine family: long-running agent loops that\n reason over memory and perform maintenance tasks on a declared shape\n (check-in interval, tool set, attached memory engine). See epic\n archie-core-yp9.\n- `mapping` owns the payload field-mapping vocabulary: binding named fields to\n JSON paths inside a captured webhook payload, and detecting drift. See\n `docs/prds/payload-field-mapping.md`.\n- `scheduling` owns the ticker engine: deciding when a scheduled job runs and\n with what concurrency. It declares `JobSource`/`Runner` contracts and owns\n no storage or delivery. See epic archie-core-1786637496320-249-be6d9a20.\n\n**Reconciliation (2026-08-31):** `curator`, `mapping`, and `scheduling` existed\nin the tree without being added to this list when their epics landed. They are\nconfirmed as legitimate, narrowly-scoped domains under the same rules as the\noriginal six — each owns one capability family, declares typed contracts, and\nhas no cross-domain reach. This list was out of date, not the tree.\n\nConversely, `identity`, `agent`, `messaging`, and `plugin` are approved target\ndomains that do not yet exist as packages. Their behaviour currently lives in\npre-migration packages (see \"Complete package destination map\" below for the\nfull inventory). Do **not** create empty stub packages for them ahead of an\nactual migration PR — an unused stub is dead code and gives no test coverage.\nEach materialises when its corresponding migration decision below is resolved\nand implemented in the same change.\n\nMemory's final package placement remains undecided pending focused review of the\nexisting implementation.\n\nThe following are approved shared or composition packages, not domains:\n\n```text\ninternal/eventbus/\ninternal/policy/\ninternal/infrastructure/\ninternal/app/archied/\ninternal/app/agentworker/\n```\n\n## Remaining migration decisions\n\n### 1. Complete package destination map\n\nEvery current package MUST receive one explicit disposition:\n\n- move into a confirmed domain;\n- move under infrastructure as an implementation of a domain-owned contract;\n- move into application composition;\n- merge with an identified duplicate;\n- remain as an approved shared application contract;\n- move to a plugin, extra, deployment, example, or generated artifact;\n- be deleted after its replacement is proven.\n\nThe largest unreviewed areas include memory, tools and capabilities, workspace\nand worktree behaviour, gates, storage, events, web UI, containers, RPC\npackages, release management, and scheduling.\n\nThe destination matrix MUST name the behaviour owner, target path, dependencies\nto remove, state and contracts to preserve, migration prerequisites, and\ndeletion criteria.\n\n**`internal/webui` / UI Service — RATIFIED Phase 3 disposition (2026-09-07).**\nThe current owner is the in-process HTTP/SSE dashboard: SPA hosting, operator\nauthentication, UI response projection, configuration reads and mutations,\ntask/control APIs, and direct adapters to State Store, channel, event/log,\nmemory, curator, workflow, forge, and Gateway runtime values. The target owner\nis a standalone UI application process; `internal/webui` remains its HTTP\ntransport adapter, while Gateway, State Store, configuration, Work Intake,\nMessaging, and other capability owners expose the contracts behind those\nroutes. The migration preserves retained route schemas, auth/CSRF behavior,\ntask/capture/mapping/binding/session continuity, SSE replay semantics, and\ncommitted generated SPA assets. Prerequisites are the ratified UI boundary,\nroute-owner inventory, narrow remote adapters, configuration/auth/readiness\ncontracts, and process/conformance tests. Remove the shared live\n`config.Holder`, direct event/log bus, daemon stopper, forge client, channel\nmanager, capability registries, composite store access, and in-process UI\nlistener only after the focused criteria in\n[`docs/prds/ui-service-boundary.md`](/dev/prds/ui-service-boundary/) pass in\nproduction composition. The old construction and listener are deleted in the\nsame cutover change; no second UI authority remains.\n\n### 2. Session and Messaging migration\n\nThe existing session implementation is the migration baseline. The migration must\ndecide:\n\n- which session and message components are retained or adapted;\n- how Agent, user, channel, Conversation, and branch ownership are added;\n- how Archie's parallel gateway session implementation is replaced;\n- how canonical Messages become immutable persisted records;\n- how existing sessions and messages are migrated;\n- how outbound delivery, retry, acknowledgement, deduplication, and failure\n semantics are represented;\n- how current Telegram, email, webhook, forge, and Jira integrations migrate to\n Messaging infrastructure adapters.\n\nThe following are already decided and MUST NOT be reopened without contradictory\nimplementation evidence:\n\n- the composite Conversation identity;\n- immutable MessageIDs;\n- typed tool-call and tool-result messages;\n- exact branch lineage through ParentConversationID and ForkMessageID;\n- parent and child transcript isolation;\n- normal scoped-memory behaviour regardless of the originating Conversation;\n- Messaging and Work Intake ownership.\n\n**Decision (2026-08-31), resolving the open questions above:**\n\n`internal/gateway/` is the \"parallel gateway session implementation\" this\nsection names. It is retained as-is for now — a big-bang rewrite of live\nsession/compression/approval/branch logic across Telegram, web UI, email, and\nwebhook channels in one change is exactly the \"convoluted to satisfy edge\ncases\" failure mode CLAUDE.md's Revert-on-Flail rule warns about, and would\nrisk silent behavioural regressions in the one path every user-facing message\ntravels through.\n\nInstead, `internal/domain/messaging/` is materialised now with the pieces\nthat are already decided and don't require touching gateway: the canonical\n`Message` and `Conversation` types (composite Conversation identity, immutable\n`MessageID`, typed tool-call/tool-result message variants, branch lineage via\n`ParentConversationID`/`ForkMessageID`). These are pure domain types with no\ngateway dependency — a new package, not a rewrite of an existing one.\n\nMigrating `internal/gateway/`'s session, compression, approval, and branch\nlogic onto these types is real, cross-channel-risk work and is **not** done in\nthis pass. It is tracked as separate, appropriately-scoped bd issues (see\n`bd list --label messaging-migration`), sequenced channel-by-channel so each\nlands with its own behavioural verification instead of one large diff no one\ncan safely review.\n\n**Amended 2026-09-15.** `internal/infrastructure/gatewayrpc.StoreClient` is the\nadapter this migration will need: it implements `messaging.SessionStore` over\nthe chat service's session-store RPCs and is proven by\n`TestStoreClientPreservesCanonicalRecords`. It is deliberately unwired.\nProduction `archied` dials the standalone gateway for `ChatContract` only\n(`composeChatContract` → `gatewayrpc.Dial` returns a `*Client`, not a\n`*StoreClient`), and the daemon's session store is in-process SQLite via\n`makeTelegramSessionStore`. The test comment that says it mirrors production\ndescribes the target, not today's tree. Kept rather than deleted because the\nmigration it serves is deferred by this very section, and re-deriving a tested\nwire adapter once that lands is pure cost.\n\n**Phase 4 Messaging Service extraction (2026-09-19, `archie-core-8cda.6.1`).**\nThe process boundary for external channels (Telegram, email, webhook) is ratified\nin [`docs/prds/messaging-service-boundary.md`](/dev/prds/messaging-service-boundary/).\nMessaging is extracted into `cmd/archie-messaging` / `internal/app/archiemessaging`,\ncalling Gateway's `messaging.ChatContract` exclusively over gRPC. Channel frontends\nhold no direct database handles, daemon state, or model runtimes.\n\n**Telegram operator surface after extraction (2026-09-19, `archie-core-8cda.6.6`).**\n`setupTelegramGateway` wired eight `telegram.Gateway` seams the daemon could\nsatisfy from its own process. Each is resolved by where the fact it needs is\nactually owned, not by where it used to be built:\n\n| Seam | Resolution |\n|---|---|\n| `Version` | Sourced from the Gateway via `ChatSnapshot.Version`. The Messaging Service never stamps or infers a build version; `cmd/archie-messaging` deliberately takes no version ldflags, so it cannot report a partial upgrade as a matched one. |\n| `Updates` (`/update`) | Built in the Messaging Service from `[chat.telegram].update_check_command` / `update_install_command` — its own configuration. `Enrich` is dropped: `componentInstallTypeEnricher` reads `[nats]`, which this service must not decode, so the check command's own reported install type stands unmodified. |\n| `UpdateReportPath` | A local state file under `work_dir`, keyed by the same `sha256(bot_user)[:8]` scheme the daemon used, so a pending report is found rather than written afresh. |\n| `ReleaseAnnouncements` | **Deliberately left nil**, for the same reason as `RunningVersions`. `releaseannounce.Announcer` announces nothing unless each `Component` carries a parseable version, and the only versions this service can obtain are the Gateway's own build stamps, not archied's. Wiring the announcer without them yields a no-op that reads as configured, so it stays unwired until component self-reporting exists. |\n| `SetShowToolCalls` | Projected from `[chat].show_tool_calls`. |\n| `Reload` | Local: re-resolves the token and allowlist from this service's own config file and overlay. Reload was never a daemon fact. |\n| `RunningVersions` | **Deliberately left nil.** It exists to turn an installer's claim into a checked one, and only a component's own compiled-in build can vouch for it. The Messaging Service knows neither archied's build nor the observed archie-agent version (`daemon.AgentStatus`), and the Gateway's own `gatewayVersion` is a different binary's stamp — reporting it as the daemon's would manufacture exactly the false success the check exists to catch. Update reports therefore relay as unverified claims until a component self-report contract exists. |\n| `Dangerous` | Stays nil. No daemon composition ever set it; `/rollback` and `/stop` reported \"not configured\" before the extraction and still do. |\n\nThe Messaging Service consequently reads `work_dir`, `bot_user`, `[chat]` and\n`[health]` in addition to the PRD's `[chat.*]` / `[services.gateway]` list.\n`[health]` is the daemon's health URL, which `releaseupdate.CommandInstaller`\npolls from outside to confirm an update came back up; the rest of the PRD's\nprohibition list is untouched.\n\n**Phase 4 messaging severance: what must move, and the decision the PRD does\nnot cover (2026-09-22, `archie-core-1ng1`).** The PRD's deletion gate cannot\npass today. `cmd/archie-messaging/architecture_test.go` does not exist, and the\ncheck that section specifies -- `go list -deps ./cmd/archie-messaging` links\nzero banned runtime packages -- reports four:\n\n| banned package | PRD category |\n| --- | --- |\n| `modernc.org/sqlite` | State Store |\n| `internal/store` | State Store |\n| `internal/domain/workflow` | Workflow engine |\n| `internal/agentexec` | Workflow engine, via `internal/domain/workflow/agent.go` |\n\nAll four arrive through one import: `internal/app/archiemessaging/run.go`\nimports `internal/app/controlplane`, whose own dependency set is exactly those\nfour. The Messaging Service uses three symbols from that package\n(`NewRPCClient`, `RuntimeChatConfig`, `ChannelSettingsKind`); the package also\ncarries the store-backed `ResourceStore` server, the `Definition` registry and\nthe workflow step vocabulary. Sever the package. Amending the gate to tolerate\n`modernc.org/sqlite` and `internal/store` is rejected: a boundary you whitelist\naround is no longer a boundary.\n\n**What moves.**\n\n- Every kind constant. They are strings with no dependency, and the client names\n them.\n- The channel-settings vocabulary and its decode: `channelSettings`,\n `telegramSettings`, `webhookChannelSettings`, `emailSettings`,\n `rateLimitSettings`, `channelDuration`, and `decodeRenamingLegacyKeys`.\n- The generic client: `Client`, `NewRPCClient`, `RuntimeConfig` with\n `runtimeConfigFrom`, `runtimeChatConfigFrom`, the other `runtime*From`\n projections, and the `resourceReader` interface.\n- The persona watch (`WatchPersonas`, `AppliedPersonas`), which needs only\n `internal/domain/agent`.\n- `executionSettingsDocument`, the document alone.\n\n**What stays, and why.**\n\n- The store-backed server: `Server`, `NewServer`, `ResourceStore`,\n `ImportConfig`, `SeedSkip`, the `Definition` registry, and every other kind's\n seed, validator and `Normalize` hook.\n- `WorkflowDefinitionsClient` and `stepRegistry`, which decode through\n `internal/domain/workflow`.\n- `WatchWorkflowExecutionSettings` and `decodeSettings`: `decodeSettings`\n returns and validates `workflow.ExecutionSettings`, so any package holding it\n links the workflow domain and therefore `internal/agentexec`. This is why the\n client cannot move as one package -- see open question 2.\n- The schedules document, whose type is `scheduling.JobSpec` in\n `internal/domain/scheduling` -- the vocabulary moved there when the legacy\n file-backed cronstore was deleted, which is also what satisfies the rule\n that a domain vocabulary package may not import infrastructure.\n- The schema derivation, which describes the documents the server advertises.\n\n**Placement as landed (2026-09-22, `archie-core-1ng1`).** One new package,\n`internal/infrastructure/controlplanerpc`, holding both the vocabulary (kinds, the\nchannel-settings document and its decode) and the client\n(`NewRPCClient`, `Query`, `RuntimeChatConfig`, the exported `ResourceReader`, the\nwire sentinels and `ClientError`). A second `internal/domain/controlplane` package\nfor the vocabulary was planned and then measured out of the design: the vocabulary\nhas exactly one consumer, the client that projects it, and the store-backed side\nreads the same definitions through type aliases rather than a copy -- so a separate\ndomain package would have been a package with one consumer and a second name for\none thing. What made that safe is a measurement, not a preference:\n`internal/app/controlplane` was the ONLY direct import of the Messaging Service\nthat dragged a banned runtime (`gatewayrpc`, `staterpc`, `applystatus`, `messaging`,\n`configuration`, `readiness` and `health` were all clean), so severing that one\ndependency is sufficient, and the severance needed no third package. Promoting the\nvocabulary to a domain package stays open for the day a second consumer appears.\n\n**Settled by the implementation.** Two of the four open questions this decision\noriginally left are answered, in both cases by measuring rather than by preference:\n\n- The client did NOT become two packages. `WatchWorkflowExecutionSettings` and\n `WatchPersonas` stayed on `internal/app/controlplane`'s `Client`, which now\n *embeds* the new one, so the daemon's validated watch is still defined once and\n the read path still has one implementation -- which is what the second package\n was for -- with one fewer package to keep in step.\n- The schedules and workflow documents stayed put, and\n `internal/domain/workflow/task` is linked deliberately: it carries the Task\n status and view vocabulary, its own dependency set holds none of the banned\n runtimes, and `cmd/archie-messaging/architecture_test.go` carries it as an\n exception checked in both directions (the same entry, for the same reason, as\n `cmd/archie-ui/architecture_test.go`).\n\n**Still open.**\n\n1. Does the server also leave `internal/app/`? `organisation.md` reserves\n `internal/app/` for wiring and composition, and a store-backed gRPC server is\n infrastructure -- but the gate does not require the move, and it would multiply\n the diff across archied's wiring and the State Store binary. Tracked as\n `archie-core-8f4n`.\n2. Does `internal/domain/workflow`'s contract/engine split happen, and when? A\n contract type (`WorkflowDefinitionCollection`) drags `internal/agentexec` only\n because `agent.go` sits beside it. Independent of this severance, tracked as\n `archie-core-inh6`, and the shape `archie-core-8cda.7` will meet again.\n3. Does the vocabulary move to a domain package when a second consumer appears?\n\n**Both consumers keep compiling.** archied imports the new vocabulary for the\nkinds it names and the new client package for `RuntimeConfig` and the persona\nwatch, and keeps `internal/app/controlplane` for the server, the registry,\n`ErrVersionConflict` and the workflow-aware pieces. The State Store binary\ncontinues to call `NewServer` and `ResourceStore` unchanged. The Messaging\nService then imports only `internal/infrastructure/controlplanerpc`,\n`gatewayrpc` and `staterpc`, so `go list -deps ./cmd/archie-messaging` loses all\nfour banned packages and the gate can land green.\n\n**Channel state to the dashboard: one administrative surface, following the\nconfig-snapshot precedent (2026-09-22, `archie-core-8cda.6.8`).** The Messaging\nService now records its channels' lifecycle -- `internal/channels/status.Manager`,\none entry per composed channel, written by each channel's own report through\n`channels.Lifecycle` -- and nothing reads that across a process boundary.\n`/api/channels` answers with nothing because `webui.Server.Channels` has no\nproduction source, and POST `/api/channels/{id}/reload` answers 501 for the same\nreason. The carrier is the State Store, and the shape is the one\n`archie-core-ymut` already established for config snapshots\n(`docs/prds/state-store-contract.md`, \"the one surface whose writer is the daemon\nand whose reader is the UI process\"): a pair of administrative RPCs, deny-by-default\nunder `authorizesTaskScopedCall`, so a task-scoped grant reaches neither.\n\nWhat that decides, so the implementation need not invent it:\n\n- A channel's state is **runtime** state, not a settings document. It does not\n belong in the `ControlPlaneService` resource kinds, which are validated,\n versioned documents seeded from config; a channel that is failed is not a\n document anybody edits.\n- The reader is the UI process, which already dials the State Store and pumps its\n cursor, so the surface needs no new transport in either direction.\n- Adding the pair changes the contract's RPC count, so\n `docs/prds/state-store-contract.md`'s grouped table gains a group in the same\n change that adds the RPCs -- the table is the contract's own index, and a\n surface it does not list is a surface nobody finds.\n\n**Reload gets its own surface, with its own lifetime (decided 2026-09-22).** Reload\nruns the other way -- the UI asks the service to reload a channel -- and the two\ncandidates were a request surface the service observes and clears, or a field on\nthe status surface the service watches. It is the first. The reasoning, recorded\nbecause it would look arbitrary later:\n\n- A request queue and a status report do not share a lifetime, so they should not\n share a surface. A pending request is transient and has an outcome; a channel's\n state is durable and has a current value.\n- A \"please reload\" field on the status resource makes READING status capable of\n RETRIGGERING an action, and makes a transient request indistinguishable from\n state the moment anything polls it.\n- It would also invert the writer/reader relationship the precedent establishes:\n the service would have to clear a field on an object whose whole contract is\n that the service reports it.\n\nSo the status surface stays writer-is-service, and the request carrier is its own\nobject with its own states.\n\n`archie-core-8cda.6.9`'s component self-report is the **same shape**: a process\nreporting something about itself for the UI to read. Whether it shares this\nsurface or gets its own is a decision for `6.9`, and the one outcome to avoid is\ntwo homes for \"what a component reports about itself\".\n\n### 3. Identity data migration\n\nThe migration must define:\n\n- the durable identity schema and repository implementation;\n- bootstrap of the permanent system identity;\n- stable IdentityIDs for existing configured Agents;\n- migration from configured identity names, empty-string ownership, and bot\n usernames;\n- preservation of historical attribution;\n- identity-aware uniqueness and transport deduplication;\n- correction of container and RPC capability resolution so execution uses the\n owning identity's bindings;\n- removal of identity-string equality from access-control behaviour.\n\nExact schema and method signatures are implementation decisions. The approved\nIdentity lifecycle and attribution semantics are fixed migration constraints.\n\n### 4. Workflow migration\n\nThe current implementation must be transformed deliberately:\n\n```text\nstore.Task -\u003e WorkflowExecution\nmutable Stage -\u003e StepExecution history\nRetryCount / Attempt -\u003e Attempt records\nworkflow.Registry -\u003e versioned Workflow definitions\ncurrent stages -\u003e WorkflowStep definitions or Workflow plugins\n```\n\nThe migration must decide:\n\n- Workflow definition and version persistence;\n- the smallest accepted-work-request contract consumed from Work Intake;\n- the enforced WorkflowExecution state machine;\n- atomic state and domain-event persistence;\n- migration of existing task rows and transition history;\n- disposition of current skill, Yaegi, and stage-plugin implementations;\n- the versioned worker-handoff contract;\n- scheduler, claim, retry, crash-recovery, and waiting-human cutover.\n\nWorkflow remains a separate domain from Agent. A user or Agent may define or\ninvoke a Workflow, but Workflow semantics and plugins remain Workflow-owned.\n\n### 5. Memory placement and storage\n\nThe four required scopes are fixed:\n\n- global shared;\n- Agent-wide;\n- user-wide;\n- Agent-user relationship.\n\nA focused review of `internal/memory`, its providers, consumers, persistence,\nand tests had these decisions:\n\n- `internal/domain/memory` versus ownership within the Agent domain — settled\n (2026-09-02, below);\n- the authoritative memory record and revision model — settled by the PRD\n (2026-09-16, below);\n- provider and infrastructure boundaries — settled by the PRD (2026-09-16,\n below);\n- retrieval and access enforcement — settled by the PRD (2026-09-16, below);\n ranked retrieval remains open (below);\n- migration of existing memory data — open: existing data is not migrated;\n- provenance representation — settled by the PRD (2026-09-16, below).\n\nConversation branches do not create another memory scope. A memory action has\nthe same scoped effect regardless of which Conversation originated it.\n\n**`internal/domain/memory` ownership — DECIDED (2026-09-02).** The typed\nengine family lives at `internal/domain/memory`\n(archie-core-1786637490069-11-e245c170), mirroring `internal/domain/curator`'s\ncontract/registrar/registry split. Not owned within the Agent domain: memory\nis consumed by more than agent turns (curators, per archie-core-1786637499636)\nand the plugin engine rule requires its own family, not a capability bolted\nonto another domain's contract.\n\n**Authoritative record and revision model for the builtin engine — DECIDED\n(2026-09-02),** scoped to unblock archie-core-1786637499480 (ship the\nfile-writing store as the default engine). This decides only the builtin\nengine's mapping onto the new `Record{ID}`/`Forget(id)` contract, not the\nmemory model in general or a future non-file engine's model:\n\n- `internal/memory/builtin.Store` is unchanged. It has no concept of a\n stable per-block ID — blocks are addressed by content substring — and nothing\n in this decision requires changing that; existing `internal/memory` tests\n keep passing unmodified, per that issue's acceptance criteria.\n- A new infrastructure adapter (`internal/infrastructure/memory`, per\n `organisation.md`'s \"infrastructure implements domain contracts\" rule — not\n under `internal/domain/memory`, which stays contract-only) wraps one\n `builtin.Store` per identity.\n- **Identity → directory:** `hex(sha256(identity))`, not the identity string\n itself. Closes the path-traversal question without an allowlist regex, and\n sidesteps collisions from identities that differ only in characters a\n filesystem treats as equivalent.\n- **Record ID:** assigned once at `Write`, independent of content, so a later\n edit elsewhere in the file can never orphan a `Forget`. Realized as a\n leading marker line (`\u003c!--mem:\u003culid\u003e--\u003e`) on the block `builtin.Store.Add`\n writes; `Query`/`List` strip it back out of `Record.Content` before\n returning, and `Forget(id)` locates the block by that unique marker via the\n store's existing substring-based `Remove`. The marker is a pragmatic,\n reversible choice — plain-text records stay human-readable and\n human-editable, at the cost of one marker line an operator editing the file\n by hand should leave alone. Revisit if/when provenance representation\n (still open, below) settles on something richer.\n- **Kind → section:** `Observation.Kind` (default `\"general\"` when empty) is\n the section name, reusing `builtin.ValidateSectionName` — so an\n invalid `Kind` fails `Write` with the same validation error `memory_edit`\n already returns for an invalid section today, not a new error shape.\n\n**Memory engine: scopes, CRUD with retained revisions, and store placement —\nDECIDED (2026-09-16).** Supersedes the 2026-09-02 \"Authoritative record and\nrevision model for the builtin engine\" entry above, whose `\u003c!--mem:\u003culid\u003e--\u003e`\nlive marker cannot carry provenance. Authority:\n[`docs/prds/memory-engine-unification.md`](/dev/prds/memory-engine-unification/)\n(Approved); the contract is `internal/domain/memory/contract.go`. This register\nsupersedes; it does not erase.\n\n- **Four typed scopes, caller-named.** `internal/domain/memory` is addressed by\n `Scope{Kind, Agent, User}` over `global`, `agent`, `user` and `agent-user`,\n with `Validate` rejecting a component present where it is not required. A\n caller names the read and write sets (`Subject.Scopes()` /\n `WritableScopes()`; global is never model-writable) and the engine applies no\n policy of its own: **access control is naming a scope, and nothing else.** An\n engine that enforced policy would need to know about channels, sessions and\n bindings, and every backend would reimplement it. `Scope.Key()` length-prefixes\n each component, so two scopes can never share a key even when an opaque\n channel-native id embeds the separator.\n- **The caller resolves identity; the engine never does.** Resolution happens in\n the gateway turn path, where the native sender id and the session are both in\n hand, and is supplied per channel by the composition. A webhook's sender id is\n a route path, not a person, and treating it as one would give a URL an\n identity. An absent user id yields global and agent scopes only: \"show none of\n it\", never a wider fallback.\n- **CRUD supersedes, never overwrites.** `Store` is `Create`/`Get`/`Query`/\n `List`/`Update`/`Forget`/`Revisions` over the four scopes. `Update` appends the\n superseded state to the scope's `HISTORY.md` before replacing the live block;\n `Update.Expected` names the revision being replaced and returns\n `ErrStaleRevision` on mismatch, making a lost update between two processes\n sharing one scope file detectable. `Forget` records a deleted revision before\n removing the live block, so a record's provenance outlives its content; `Get`\n and `Forget` take a scope, because an id alone does not locate one.\n- **Provenance representation.** The live marker becomes JSON, since a\n `\u003c!--mem:\u003culid\u003e--\u003e` line cannot carry provenance, and `Record` retains the\n provenance `agent-system.md` requires — scope, author, originating user,\n source, and revision history. `Metadata` is dropped: it never round-tripped.\n- **Store and scanner placement.** The markdown store relocates to\n `internal/infrastructure/memory/builtin`, its correct home under\n `organisation.md`'s \"a package owns its on-disk format end-to-end\". The content\n scanner moves into `internal/domain/memory` and is applied in `Create` and\n `Update` — the single choke point every producer crosses, so curator-extracted\n content is covered too.\n\n**Open in this section.** Ranked retrieval is deferred: `Query.Text` stays in\nthe contract, unused by the prompt path. Existing memory data is not migrated:\n`\u003cworkDir\u003e/memory` and `\u003cworkDir\u003e/memory-engine` are left on disk, unreferenced.\nThe PRD leaves these undetermined and they remain open here: group-chat\nextraction, where the curator fails closed on mixed-sender sessions; two\nprocesses (`archied` and `archie-gateway`) holding one scope file, where\n`Update.Expected` detects but does not prevent a lost block; unbounded\n`HISTORY.md` with no compaction policy; whether the per-scope size bound is\nright; and whether the dashboard should carry a user identity.\n\n### 6. Runtime and process boundaries — DECIDED\n\n`archied` is the native resident orchestration and interactive-chat process.\nEvery autonomous repository workflow crosses one full-task core-NATS handoff\ninto a task-scoped `archie-agent` container. Embedded versus external NATS\nchanges broker deployment only; it never selects an executor. The former host\nin-process runner, per-stage NATS protocol, and subprocess runner are removed.\n\nForge, store, and worktree authority remains in `archied` and is exposed to the\ntask container through scoped RPC services. The worker runs the workflow and\nits worker-local ai-sdk loop; it does not receive forge credentials or direct\nstore access. See `docs/prds/embedded-nats.md`.\n\n**UI Service boundary — RATIFIED (2026-09-07).** Phase 3 has a focused\nauthority record in [`docs/prds/ui-service-boundary.md`](/dev/prds/ui-service-boundary/).\nThe UI Service owns HTTP/SPA delivery, UI DTOs, browser authentication, and\nits own endpoint/credential/readiness settings. It consumes Gateway and State\nStore contracts and does not receive `config.Holder`, daemon pointers, SQL or\nstore implementations, workflow registries, forge/channel/model runtimes, or\nresolved secrets. The current `internal/webui` implementation remains inside\n`archied` until the ratified migration gates prove route parity, adapter\nselection, failure/readiness behavior, and deletion of the shared-holder and\ndirect-access paths. The extraction cutover removes the in-process UI\nlistener in the same change; there is no dual-live UI authority.\n\n**Dashboard live event delivery — DECIDED (2026-09-09, `archie-core-za9f`).**\nThe extracted UI receives live activity by polling the State Store's existing\n`EventsSince` cursor, not by a new push-subscription RPC. No proto change, no\ncontract amendment beyond this record.\n\nThe UI process runs one event pump. It primes its watermark by paging\n`EventsSince` to the end at startup without delivering, then polls\n`EventsSince(watermark, limit)` on an interval and hands each new event to\n`webui.Server.Broadcast`, exactly where `internal/app/archied/main.go`'s\n`persistAndBroadcastEvents` hands events to it today. One pump serves every\nconnected browser, so cost is one indexed query per interval regardless of\nviewer count, and `Broadcast` is a no-op when nobody is watching.\n\nThree properties make the cursor sufficient, and a push hub unnecessary:\n\n- **The cursor cannot skip.** `internal/store/store.go` opens the database with\n `SetMaxOpenConns(1)`, so writes serialise and a row's `id` is assigned in\n commit order. There is no window in which a lower id becomes visible after a\n higher one, which is the failure mode that normally rules out polling an\n autoincrement cursor. SSE catch-up already depends on this same property.\n- **A broadcast is a wakeup, not the payload.** `sseStream.drain` treats each\n broadcast event as a signal to fill the persisted gap ahead of it via\n `catchUp`, then delivers the event only if catch-up did not already cover\n it, deduplicating on the `since` watermark. Substituting a poller for the\n in-process bus therefore changes no SSE semantics, and needs no change to\n `internal/webui/sse.go`. It also means a broadcast dropped to a stalled\n client self-heals on the next one.\n- **The table is the only complete fan-out point.** Events reach the store\n through several paths that never touch the daemon's bus, including the\n audit event `store.ArchiveTask` writes inside its transaction\n (`internal/store/store.go`) and the binding-dispatch failure\n `internal/daemon/daemon.go` inserts directly. A push hub would have to be\n hooked at every write site and at post-commit, and would silently miss any\n site added later that forgets to notify. Reading the table misses nothing.\n\nThe cost is one poll interval of latency on the activity feed, default 1s and\noperator-tunable. That is accepted: the feed reports task lifecycle history,\nand an operator action already gets its own synchronous HTTP response.\n\nThis resolves the `/events` row of the route-owner inventory in\n[`ui-service-boundary.md`](/dev/prds/ui-service-boundary/). `/api/logs` and\n`/api/logs/stream` are not resolved by it, they read the daemon's in-process\ndiagnostic feed and remain host-local (see `archie-core-8cda.5.4`).\n\nRevisit only if sub-second feed latency becomes a requirement, or if a second\nconsumer needs the same stream, at which point the fan-out belongs in the State\nStore process behind a `SubscribeEvents` RPC and this record is superseded.\n\n**Dashboard configuration page — DECIDED (2026-09-09, `archie-core-ymut`).**\nThe configuration page becomes read-only in the UI process for Phase 3. Reads\ncross the boundary as a published snapshot; writes are descoped, not\ncontracted. This amends the ratified boundary, which assumed a single admin\ncontract carrying both.\n\n**Reads.** `webui.ConfigView` is already the whole answer. It is a UI-owned,\nsecret-free value projection built in `handleConfig`: `ProviderView` carries\nthe API key's environment variable name and a `Configured` boolean, never a\nvalue, and `RepoView` is repository metadata only. It is already the exact\nshape the browser receives. So the daemon publishes that projection, plus\nprovenance, reload status and the overridden-key list, as a snapshot the UI\nreads; nothing new has to be designed, only moved. Two State Store RPCs carry\nit, a `PutConfigSnapshot` restricted to the administrative token and a\n`GetConfigSnapshot` the UI calls. The daemon republishes on boot and after\neach successful reload, which is precisely when the projection changes.\n\nThe State Store is the right holder because a published configuration snapshot\nis shared state, it is already dialled by the UI, and it costs no new listener,\nport, token or interceptor. The alternative, a config admin service on\n`archied`, would make the daemon a gRPC server for the first time. Today it is\npurely a client of Gateway and State Store; `grpc.NewServer` appears only in\n`RunGateway` and `RunStateStore`, which are the standalone binaries' entry\npoints, not the daemon's.\n\n**Writes.** `PATCH /api/config`, `POST /api/config/reset` and\n`PATCH /api/config/repos/{owner}/{name}` answer 503 in the UI process, and the\nSPA hides those controls rather than offering a button that fails. Editing is\n`config.toml` plus reload, which is already the only path for every setting\noutside the runtime-tunable allowlist.\n\nThis is a deliberate feature reduction and worth naming as one. The write path\nis not hard because of the transport; it is hard because the policy is\ndaemon-local and stays that way: `installUpdateConfigHandler` denies\nnon-runtime-tunable keys, applies a dotted overlay to a clone of the published\nconfig, validates the materialised result, writes overlay rows, and rebuilds\nprovenance. Contracting that means either moving the policy, which the\nboundary forbids, or exposing it verbatim as remote procedure calls, which\nbuilds a Configuration Service inside a migration bead. Neither is work this\nphase should be doing to keep one page's buttons alive.\n\nRevisit when dashboard configuration editing is picked up as a feature in its\nown right (`archie-core-1786637498420-327`, still open). At that point it gets\nan admin contract designed as a feature, and the Configuration Service this\ndocument already anticipates is its natural owner. Until then the UI renders\nconfiguration and does not write it, which is what the boundary always said\nabout ownership even when it assumed a wider contract.\n\nAmended 2026-09-11. This descope is a deferral, not a decision that the\ndashboard should be read-only. Name it that way in review: the daemon-side\npolicy was built and tested, then deleted by the cutover (`e037b95` removed\n`internal/app/archied/config_update.go` and its tests), so reviving it is\nrecoverable work, not a rewrite. `archie-core-j28m` carries the remaining\nquestion -- which process serves the policy and what transport carries the\nwrite -- and blocks the feature bead. The handlers, their error sentinels and\nthe `UpdateConfig`/`ResetConfig`/`UpdateRepoField` seams on `webui.Server` are\nkept deliberately for it; `ConfigView.Editable` follows whichever process holds\n`UpdateConfig`, so wiring that one field makes the page editable\n(`archie-core-ml30`).\n\nImplementation belongs to `archie-core-8cda.5.4`, the cutover that removes the\ndaemon's own dashboard. Nothing here is built yet.\n\n**Superseded 2026-09-21 by the runtime control plane.** The open question this\ndescope left -- which process serves the policy and what transport carries the\nwrite -- was answered by `docs/prds/runtime-control-plane.md`: neither a daemon\nadmin service nor a revival of the daemon-local policy. `ControlPlaneService` on\nthe State Store's existing gRPC server owns every database-backed setting, and\neach feature validates its own resource, so there is no daemon-local policy left\nto contract. `archie-core-j28m` is closed.\n\nConsequently the three write seams described above no longer exist.\n`PATCH /api/config`, `POST /api/config/reset` and\n`PATCH /api/config/repos/{owner}/{name}`, the `UpdateConfig`/`ResetConfig`/\n`UpdateRepoField` fields on `webui.Server`, and `ConfigView.Editable` were\ndeleted in `31622ec5` along with the runtime overlay store they wrote to.\n`GET /api/config` remains a read of the published projection, as this decision\nalways intended.\n\n### 7. Shared mechanics\n\nThe following cross-domain mechanics require exact contracts before dependent\npackages migrate:\n\n- event-bus delivery and failure semantics;\n- policy evaluation API, composition, and precedence;\n- configuration candidate preparation, health validation, atomic promotion,\n observation, and rollback;\n- service-specific health, bounded remediation, isolation, and escalation;\n- changelog association and grouping.\n\nThese shared mechanisms do not acquire ownership of domain meaning. Domains\ncontinue to own their commands, events, policies, settings, and consequences.\n\nThe Go documentation generator, its output placement, and drift checking are\ndefined in `generated-documentation.md`. The VitePress site and its Pages\ndeployment were removed on 2026-09-12 as an unnecessary build step; `docs/` is\nrepository documentation only. The rendering and publishing surface is an open\ndecision, and so are the renderer's page templates and any developer commands\nor CI sequence that would accompany one. Adapting the generator and implementing\nArchie's normalized documentation model remain migration work rather than open\nproduct architecture.\n\n**Amended 2026-09-15.** `internal/infrastructure/servicediscovery/nats`\nimplements the `servicediscovery.ServiceRegistry` contract (Connect, Register,\nResolve, Watch) over NATS KV and is fully tested, but no production code\nconstructs it and no config surface exposes it. This is a deliberate deferral,\nnot an oversight: service-to-service addressing belongs to the runtime and\nprocess boundary work in section 6, which currently pins container addressing\nby injecting the daemon's startup NATS URL rather than resolving through a\nregistry. The scaffold is kept for that work; re-derive it only if the\naddressing model changes.\n\n### 8. Cutover sequence\n\nThe final migration plan must order:\n\n- compatibility adapters;\n- schema migrations and data backfills;\n- any required dual-read or transitional paths;\n- domain-by-domain package replacement;\n- production wiring changes;\n- legacy package removal;\n- architecture tests preventing dependency regression;\n- focused, integration, race, and full quality gates.\n\nEach legacy package requires objective removal criteria. No compatibility path\nmay become an indefinite second implementation.\n\n\n## Completion criteria\n\nMigration design is complete when:\n\n- every current package has one approved owner and destination;\n- every mutable record has one authoritative owner and migration path;\n- all external and cross-process contracts have versioned owners;\n- existing behaviour and data have explicit parity requirements;\n- process boundaries have evidence-backed justification;\n- the cutover order preserves a working application;\n- legacy deletion criteria and deterministic architecture tests are defined;\n- no unresolved decision requires an implementer to invent domain semantics.\n\nExact Go APIs, SQL statements, and mechanical refactoring steps MAY be finalized\nduring implementation when they do not change these semantics.\n\n## Recorded migration position (2026-07-29)\n\nRecovered 2026-08-09 from the pre-migration issue tracker. This section records\ndecisions the maintainer approved that the repository did not otherwise capture.\n\n**Completed:** `internal/infrastructure/eventbus/nats` (rebuilt, not relocated);\n`internal/infrastructure/configuration` (loading moved out of `internal/config`,\nwhich is now types-only and does no I/O); `internal/eventbus` (broker-neutral\ncontract); `internal/domain/workintake` (the first `internal/domain/` package,\nowning `TaskEnvelope`, the routing `Kind`, the label vocabulary, and task\nsubjects).\n\n**DECISION — private input DTO deferred (approved).**\n`configuration.Document.Config` is still `internal/config.Config` rather than the\nprivate per-domain input DTO this document mandates. Building an ~800-line\nparallel type tree while every consumer still reads `config.Config` would mean\ntwo shapes kept in sync by hand, with no consumer for the new one. It is named as\na field so the compromise is visible at each use site, with the deletion\ncondition recorded in that package's `doc.go`. Do this **with** the\n`internal/config` dissolution, not ahead of it.\n\n**Agreed next: `internal/app/{archied,agentworker}`.** `run()` in\n`cmd/archied/main.go` is ~460 lines, against `organisation.md`'s rule that\n`cmd/*` must not contain substantive wiring or act as a service locator. It is\nalso the last non-infrastructure holder of the NATS SDK, and it is where the\nconfig DTO translation will land, so it unblocks the dissolution above.\n\n**Method constraint.** Do not write a separate recorded package-review document\nper area when the decisions are already dictated by\n`dependencies-and-contracts.md`. That is the ceremony pattern this project\ndeliberately avoids.\n\n## OPEN — `[chat.*]` config section naming\n\n`configuration.md:160` already states the position: `ChatConfig` holds \"separate\nchannel instance settings... there is no chat-wide settings owner\". The config\nsection name has not followed, so a settings block named for a product surface\nowns what the architecture calls channel state.\n\nTwo candidates, neither settled: `[channel.*]` and `[messaging.channel.*]`. The\nsecond reads as a domain path, and no config section in this repo is currently\nnamed after a domain package, so adopting it sets a precedent for every other\nsection rather than fixing one name.\n\nBlocked on a maintainer decision, not on evidence. This is a rename with an\noperator-visible TOML break, so it should land with the `internal/config`\ndissolution recorded above rather than on its own.\n" }, { "url": "/dev/architecture/observability/", From 21bc8d156d2f97c126337a4a61ffd08ec84aa082 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Wed, 23 Sep 2026 03:06:25 +1000 Subject: [PATCH 14/20] feat(store): generalise the event feed cursor to a total order EventsSince no longer depends on events.id being monotonic. The feed now pages a total order over (at, id): at is a fixed-width, lexicographically sortable UTC timestamp (storecontract.EventCursorLayout) and id is only a deterministic tie-break for rows sharing one timestamp. The cursor is an opaque string on the wire (EventsSinceRequest.cursor), carried verbatim as the SSE id: field and echoed back via Last-Event-ID, so EventSource resume needs no client change. A new idempotent migration normalises pre-existing RFC3339Nano at values to the fixed-width layout before finishTaskMigration, and idx_events_at backs the new query. Legacy integer and garbage cursors degrade to "from the beginning", preserving the old failed-ParseInt-yields-0 behaviour. Amends the "Dashboard live event delivery" decision's cursor-cannot-skip justification, which previously rested on commit-order id assignment. --- internal/contracts/state/v1/state.pb.go | 21 +-- internal/daemon/dispatch_bindings_test.go | 2 +- internal/domain/storecontract/cursor.go | 54 +++++++ .../domain/storecontract/storecontract.go | 2 +- internal/infrastructure/staterpc/client.go | 4 +- .../staterpc/conformance_test.go | 2 +- internal/infrastructure/staterpc/server.go | 2 +- internal/store/events.go | 39 ++++- internal/store/store.go | 66 ++++++++ internal/store/store_test.go | 150 +++++++++++++++-- internal/webui/api_tasks_actions_test.go | 2 +- internal/webui/event_pump.go | 27 ++-- internal/webui/event_pump_test.go | 15 +- internal/webui/sse.go | 61 +++---- internal/webui/sse_test.go | 152 ++++++++++++------ internal/webui/webui_test.go | 24 +-- proto/state/v1/state.proto | 5 +- 17 files changed, 491 insertions(+), 137 deletions(-) create mode 100644 internal/domain/storecontract/cursor.go diff --git a/internal/contracts/state/v1/state.pb.go b/internal/contracts/state/v1/state.pb.go index d96463dec..cacdc3308 100644 --- a/internal/contracts/state/v1/state.pb.go +++ b/internal/contracts/state/v1/state.pb.go @@ -4133,9 +4133,12 @@ func (x *InsertEventResponse) GetId() int64 { } type EventsSinceRequest struct { - state protoimpl.MessageState `protogen:"open.v1"` - SinceId int64 `protobuf:"varint,1,opt,name=since_id,json=sinceId,proto3" json:"since_id,omitempty"` - Limit int64 `protobuf:"varint,2,opt,name=limit,proto3" json:"limit,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + // cursor is the opaque resume cursor returned by the previous page (the SSE + // id: field, echoed back as Last-Event-ID). Empty means "from the + // beginning". + Cursor string `protobuf:"bytes,1,opt,name=cursor,proto3" json:"cursor,omitempty"` + Limit int64 `protobuf:"varint,2,opt,name=limit,proto3" json:"limit,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -4170,11 +4173,11 @@ func (*EventsSinceRequest) Descriptor() ([]byte, []int) { return file_state_v1_state_proto_rawDescGZIP(), []int{70} } -func (x *EventsSinceRequest) GetSinceId() int64 { +func (x *EventsSinceRequest) GetCursor() string { if x != nil { - return x.SinceId + return x.Cursor } - return 0 + return "" } func (x *EventsSinceRequest) GetLimit() int64 { @@ -7925,9 +7928,9 @@ const file_state_v1_state_proto_rawDesc = "" + "\x12InsertEventRequest\x12%\n" + "\x05event\x18\x01 \x01(\v2\x0f.state.v1.EventR\x05event\"%\n" + "\x13InsertEventResponse\x12\x0e\n" + - "\x02id\x18\x01 \x01(\x03R\x02id\"E\n" + - "\x12EventsSinceRequest\x12\x19\n" + - "\bsince_id\x18\x01 \x01(\x03R\asinceId\x12\x14\n" + + "\x02id\x18\x01 \x01(\x03R\x02id\"B\n" + + "\x12EventsSinceRequest\x12\x16\n" + + "\x06cursor\x18\x01 \x01(\tR\x06cursor\x12\x14\n" + "\x05limit\x18\x02 \x01(\x03R\x05limit\">\n" + "\x13EventsSinceResponse\x12'\n" + "\x06events\x18\x01 \x03(\v2\x0f.state.v1.EventR\x06events\",\n" + diff --git a/internal/daemon/dispatch_bindings_test.go b/internal/daemon/dispatch_bindings_test.go index 63e16501b..3d2eb59c5 100644 --- a/internal/daemon/dispatch_bindings_test.go +++ b/internal/daemon/dispatch_bindings_test.go @@ -171,7 +171,7 @@ func TestDispatchBindingsSkipsWhenResolveFails(t *testing.T) { if len(tasks) != 0 { t.Fatalf("required-field failure spawned %d task(s), want 0", len(tasks)) } - events, err := s.EventsSince(t.Context(), 0, 100) + events, err := s.EventsSince(t.Context(), "", 100) if err != nil { t.Fatalf("EventsSince: %v", err) } diff --git a/internal/domain/storecontract/cursor.go b/internal/domain/storecontract/cursor.go new file mode 100644 index 000000000..62e300baa --- /dev/null +++ b/internal/domain/storecontract/cursor.go @@ -0,0 +1,54 @@ +package storecontract + +import ( + "fmt" + "strconv" + "strings" + "time" +) + +// EventCursorLayout is the fixed-width UTC layout every persisted event's at +// column and every EventsSince cursor is built with. Fixed width is the whole +// point: time.RFC3339Nano trims trailing zeros, so a whole-second row +// ("...05Z") and a full-precision row ("...05.123456789Z") would compare out +// of chronological order as strings. With nine fractional digits always +// present, lexicographic order is chronological order. It mirrors edastore's +// capturedAtLayout, which exists for the same reason; the two stores now share +// this rule. +const EventCursorLayout = "2006-01-02T15:04:05.000000000Z" + +// cursorSeparator joins the sort key and the row id inside a cursor. The sort +// key never contains it, so splitting on the last occurrence is unambiguous. +const cursorSeparator = "|" + +// cursorIDWidth zero-pads the row id so the whole cursor is lexicographically +// sortable: the fixed-width sort key is followed by a fixed-width tie-break, +// and string comparison of two cursors is the total order (key, id). +const cursorIDWidth = 20 + +// EventCursor returns the opaque resume cursor for an event that happened at +// at with row id id. The cursor is a string so EventSource can carry it +// verbatim in its id: field and echo it back as the Last-Event-ID header; it +// is opaque to every consumer except EventsSince. +func EventCursor(at time.Time, id int64) string { + return at.UTC().Format(EventCursorLayout) + cursorSeparator + fmt.Sprintf("%0*d", cursorIDWidth, id) +} + +// ParseEventCursor splits a cursor back into its sort key (the fixed-width at +// string) and row id. ok is false for an empty or malformed cursor, which +// means "from the beginning" -- the store must not treat a bad resume point as +// an error worth failing a feed over. +func ParseEventCursor(cursor string) (at string, id int64, ok bool) { + if cursor == "" { + return "", 0, false + } + i := strings.LastIndex(cursor, cursorSeparator) + if i < 0 { + return "", 0, false + } + id, err := strconv.ParseInt(cursor[i+1:], 10, 64) + if err != nil { + return "", 0, false + } + return cursor[:i], id, true +} diff --git a/internal/domain/storecontract/storecontract.go b/internal/domain/storecontract/storecontract.go index 2729d9638..d8c199887 100644 --- a/internal/domain/storecontract/storecontract.go +++ b/internal/domain/storecontract/storecontract.go @@ -80,7 +80,7 @@ type TaskQueries interface { // TaskEvents groups observability and lifecycle methods. type TaskEvents interface { InsertEvent(ctx context.Context, e events.Event) (int64, error) - EventsSince(ctx context.Context, sinceID int64, limit int) ([]events.Event, error) + EventsSince(ctx context.Context, cursor string, limit int) ([]events.Event, error) TaskEvents(ctx context.Context, taskID int64) ([]events.Event, error) WorkflowStats(ctx context.Context) ([]WorkflowStat, error) StageStats(ctx context.Context) ([]StageStat, error) diff --git a/internal/infrastructure/staterpc/client.go b/internal/infrastructure/staterpc/client.go index 3037918e8..9b18c5a7e 100644 --- a/internal/infrastructure/staterpc/client.go +++ b/internal/infrastructure/staterpc/client.go @@ -215,8 +215,8 @@ func (c *Client) InsertEvent(ctx context.Context, e events.Event) (int64, error) return r.Id, nil } -func (c *Client) EventsSince(ctx context.Context, sinceID int64, limit int) ([]events.Event, error) { - r, err := c.client.EventsSince(ctx, &pb.EventsSinceRequest{SinceId: sinceID, Limit: int64(limit)}) +func (c *Client) EventsSince(ctx context.Context, cursor string, limit int) ([]events.Event, error) { + r, err := c.client.EventsSince(ctx, &pb.EventsSinceRequest{Cursor: cursor, Limit: int64(limit)}) if err != nil { return nil, unmapError(err) } diff --git a/internal/infrastructure/staterpc/conformance_test.go b/internal/infrastructure/staterpc/conformance_test.go index b2a322248..7d4f22b1a 100644 --- a/internal/infrastructure/staterpc/conformance_test.go +++ b/internal/infrastructure/staterpc/conformance_test.go @@ -232,7 +232,7 @@ func TestStateStoreConformance(t *testing.T) { if !ok || doc["bot_user"] != "archie" { t.Errorf("config_captured payload = %#v, want the decoded document under its own key", evs[1].Data) } - if _, err := c.EventsSince(ctx, 0, 10); err != nil { + if _, err := c.EventsSince(ctx, "", 10); err != nil { t.Fatalf("EventsSince: %v", err) } if _, err := c.WorkflowStats(ctx); err != nil { diff --git a/internal/infrastructure/staterpc/server.go b/internal/infrastructure/staterpc/server.go index 68b93a3c3..11097ef05 100644 --- a/internal/infrastructure/staterpc/server.go +++ b/internal/infrastructure/staterpc/server.go @@ -283,7 +283,7 @@ func (s *server) InsertEvent(ctx context.Context, r *pb.InsertEventRequest) (*pb } func (s *server) EventsSince(ctx context.Context, r *pb.EventsSinceRequest) (*pb.EventsSinceResponse, error) { - evs, err := s.deps.Tasks.EventsSince(ctx, r.SinceId, int(r.Limit)) + evs, err := s.deps.Tasks.EventsSince(ctx, r.Cursor, int(r.Limit)) if err != nil { return nil, s.logErr("EventsSince", err) } diff --git a/internal/store/events.go b/internal/store/events.go index 081672db1..d860ef832 100644 --- a/internal/store/events.go +++ b/internal/store/events.go @@ -8,6 +8,7 @@ import ( "fmt" "time" + "github.com/samcharles93/archie-core/internal/domain/storecontract" "github.com/samcharles93/archie-core/internal/domain/workflow" "github.com/samcharles93/archie-core/internal/events" ) @@ -31,6 +32,7 @@ CREATE TABLE IF NOT EXISTS events ( ); CREATE INDEX IF NOT EXISTS idx_events_task ON events(task_id, id); CREATE INDEX IF NOT EXISTS idx_events_kind ON events(kind, id); +CREATE INDEX IF NOT EXISTS idx_events_at ON events(at, id); ` // InsertEvent appends an event to the log and returns its row id. @@ -53,7 +55,7 @@ func insertEvent(ctx context.Context, execer eventExecer, e events.Event) (int64 res, err := execer.ExecContext(ctx, ` INSERT INTO events (at, kind, task_id, repo, issue, workflow, stage, attempt, actor_id, actor_kind, principal_id, detail, data) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, - e.At.UTC().Format(time.RFC3339Nano), e.Kind, e.TaskID, e.Repo, e.Issue, + e.At.UTC().Format(storecontract.EventCursorLayout), e.Kind, e.TaskID, e.Repo, e.Issue, e.Workflow, e.Stage, e.Attempt, e.ActorID, e.ActorKind, e.PrincipalID, clip(e.Detail, 4000), string(data)) if err != nil { @@ -74,7 +76,7 @@ func scanEvents(rows *sql.Rows) (out []events.Event, retErr error) { &e.PrincipalID, &e.Detail, &data); err != nil { return nil, err } - e.At, _ = time.Parse(time.RFC3339Nano, at) + e.At = parseEventAt(at) _ = json.Unmarshal([]byte(data), &e.Data) out = append(out, e) } @@ -83,17 +85,42 @@ func scanEvents(rows *sql.Rows) (out []events.Event, retErr error) { const eventCols = "id, at, kind, task_id, repo, issue, workflow, stage, attempt, actor_id, actor_kind, principal_id, detail, data" -// EventsSince returns up to limit events with id > sinceID, oldest -// first -- SSE catch-up and the live feed. -func (s *Store) EventsSince(ctx context.Context, sinceID int64, limit int) ([]events.Event, error) { +// EventsSince returns up to limit events after the opaque cursor, oldest +// first -- SSE catch-up and the live feed. The cursor carries the total +// order (at, id): at is the fixed-width sort key and id only breaks ties, so +// a non-monotonic id scheme can neither skip nor repeat an event. An empty or +// malformed cursor means "from the beginning". +func (s *Store) EventsSince(ctx context.Context, cursor string, limit int) ([]events.Event, error) { + at, id, ok := storecontract.ParseEventCursor(cursor) + if !ok { + rows, err := s.db.QueryContext(ctx, + `SELECT `+eventCols+` FROM events ORDER BY at, id LIMIT ?`, limit) + if err != nil { + return nil, err + } + return scanEvents(rows) + } rows, err := s.db.QueryContext(ctx, - `SELECT `+eventCols+` FROM events WHERE id > ? ORDER BY id LIMIT ?`, sinceID, limit) + `SELECT `+eventCols+` FROM events WHERE at > ? OR (at = ? AND id > ?) ORDER BY at, id LIMIT ?`, + at, at, id, limit) if err != nil { return nil, err } return scanEvents(rows) } +// parseEventAt reads the stored timestamp, tolerating both the fixed-width +// layout new rows are written in and the RFC3339Nano form older rows carried +// before the migration normalised them. +func parseEventAt(raw string) time.Time { + for _, layout := range []string{storecontract.EventCursorLayout, time.RFC3339Nano, time.RFC3339} { + if t, err := time.Parse(layout, raw); err == nil { + return t + } + } + return time.Time{} +} + // TaskEvents returns a task's full timeline, oldest first. func (s *Store) TaskEvents(ctx context.Context, taskID int64) ([]events.Event, error) { rows, err := s.db.QueryContext(ctx, diff --git a/internal/store/store.go b/internal/store/store.go index 066ead888..e307d6a2b 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -15,6 +15,7 @@ import ( _ "modernc.org/sqlite" + "github.com/samcharles93/archie-core/internal/domain/storecontract" "github.com/samcharles93/archie-core/internal/domain/workflow" "github.com/samcharles93/archie-core/internal/events" ) @@ -131,6 +132,9 @@ func migrateTasks(ctx context.Context, db *sql.DB) error { if err := migrateResourceHistoryToPerKindRequestIDs(ctx, tx); err != nil { return err } + if err := migrateEventTimestamps(ctx, tx); err != nil { + return err + } return finishTaskMigration(ctx, tx, columns) } @@ -172,6 +176,68 @@ func migrateResourceHistoryToPerKindRequestIDs(ctx context.Context, tx *sql.Tx) return nil } +// migrateEventTimestamps normalises the events.at column to the fixed-width +// layout the EventsSince cursor sorts on. Rows written before the cursor +// change used time.RFC3339Nano, which trims trailing zeros, so a whole-second +// row (20 chars) and a full-precision row (30 chars) would compare out of +// chronological order as strings. The ALTER-only migration arms never UPDATE +// values, so this is a separate idempotent step (the v4 precedent): it +// re-evaluates on every open, detects an already-normalised file by length, +// and re-runs cleanly on one that has never been migrated. +func migrateEventTimestamps(ctx context.Context, tx *sql.Tx) error { + updates, err := scanLegacyEventTimestamps(ctx, tx) + if err != nil { + return err + } + for _, u := range updates { + if _, err := tx.ExecContext(ctx, `UPDATE events SET at = ? WHERE id = ?`, u.at, u.id); err != nil { + return err + } + } + return nil +} + +// eventTimestampUpdate is one event row whose at predates the fixed-width +// layout, re-rendered into that layout. +type eventTimestampUpdate struct { + id int64 + at string +} + +// scanLegacyEventTimestamps reads every event row whose at predates the +// fixed-width layout, parses it, and re-renders it. The read cursor is closed +// by the defer before this function returns, so the caller's UPDATEs run on a +// free connection -- the store opens with SetMaxOpenConns(1), and a still-open +// read cursor would hold that single connection and deadlock the writes. +func scanLegacyEventTimestamps(ctx context.Context, tx *sql.Tx) ([]eventTimestampUpdate, error) { + // A fixed-width row is exactly len(EventCursorLayout) characters; anything + // else predates the layout and needs re-formatting. + fixed := strings.Repeat("?", len(storecontract.EventCursorLayout)) + rows, err := tx.QueryContext(ctx, `SELECT id, at FROM events WHERE at NOT GLOB '`+fixed+`'`) + if err != nil { + return nil, err + } + defer rows.Close() + + var updates []eventTimestampUpdate + for rows.Next() { + var u eventTimestampUpdate + if err := rows.Scan(&u.id, &u.at); err != nil { + return nil, err + } + parsed, err := time.Parse(time.RFC3339Nano, u.at) + if err != nil { + return nil, fmt.Errorf("normalise event timestamp %q: %w", u.at, err) + } + u.at = parsed.UTC().Format(storecontract.EventCursorLayout) + updates = append(updates, u) + } + if err := rows.Err(); err != nil { + return nil, err + } + return updates, nil +} + func finishTaskMigration(ctx context.Context, tx *sql.Tx, columns map[string]bool) error { if columns["owner"] && columns["repo"] && columns["pr_number"] { if _, err := tx.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_tasks_pr ON tasks(owner, repo, pr_number)`); err != nil { diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 5d1a24eeb..6d8425cd7 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -12,6 +12,7 @@ import ( "time" "unicode/utf8" + "github.com/samcharles93/archie-core/internal/domain/storecontract" "github.com/samcharles93/archie-core/internal/domain/workflow" "github.com/samcharles93/archie-core/internal/events" ) @@ -425,11 +426,10 @@ func TestStoreOperationsHonorContextCancellation(t *testing.T) { func TestOversizedEventPayload(t *testing.T) { s := openTest(t) detail := strings.Repeat("x", 5000) - id, err := s.InsertEvent(t.Context(), events.Event{Kind: events.KindLog, Detail: detail}) - if err != nil { + if _, err := s.InsertEvent(t.Context(), events.Event{Kind: events.KindLog, Detail: detail}); err != nil { t.Fatal(err) } - got, err := s.EventsSince(t.Context(), id-1, 1) + got, err := s.EventsSince(t.Context(), "", 1) if err != nil { t.Fatal(err) } @@ -440,14 +440,13 @@ func TestOversizedEventPayload(t *testing.T) { func TestInsertEventWithUnmarshalableData(t *testing.T) { s := openTest(t) - id, err := s.InsertEvent(t.Context(), events.Event{ + if _, err := s.InsertEvent(t.Context(), events.Event{ Kind: events.KindLog, Data: map[string]any{"channel": make(chan int)}, - }) - if err != nil { + }); err != nil { t.Fatal(err) } - got, err := s.EventsSince(t.Context(), id-1, 1) + got, err := s.EventsSince(t.Context(), "", 1) if err != nil { t.Fatal(err) } @@ -473,6 +472,133 @@ func openTest(t *testing.T) *Store { return s } +// TestEventsSinceOrdersByAt pins the total-order invariant: the feed cursor +// must page by at, not by row id. Inserting a later-timestamped event before +// an earlier one must still return the earlier one first, because row id is +// insertion order, not chronology. +func TestEventsSinceOrdersByAt(t *testing.T) { + s := openTest(t) + ctx := t.Context() + later := time.Date(2026, 1, 1, 0, 0, 1, 0, time.UTC) + earlier := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + if _, err := s.InsertEvent(ctx, events.Event{Kind: "log", At: later, Detail: "later"}); err != nil { + t.Fatal(err) + } + if _, err := s.InsertEvent(ctx, events.Event{Kind: "log", At: earlier, Detail: "earlier"}); err != nil { + t.Fatal(err) + } + + got, err := s.EventsSince(ctx, "", 10) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 { + t.Fatalf("EventsSince returned %d events, want 2", len(got)) + } + if got[0].Detail != "earlier" || got[1].Detail != "later" { + t.Fatalf("order = %q then %q; want earlier then later (at order, not insertion order)", got[0].Detail, got[1].Detail) + } +} + +// TestInsertEventWritesFixedWidthTimeKey pins the write format the total +// order depends on: a whole-second timestamp and a full-precision one must +// render to the same length, or lexicographic order stops being chronological +// order. time.RFC3339Nano trims trailing zeros, so today this fails. +func TestInsertEventWritesFixedWidthTimeKey(t *testing.T) { + s := openTest(t) + ctx := t.Context() + whole := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + precise := time.Date(2026, 1, 1, 0, 0, 0, 123456789, time.UTC) + for _, e := range []events.Event{ + {Kind: "log", At: whole, Detail: "whole"}, + {Kind: "log", At: precise, Detail: "precise"}, + } { + if _, err := s.InsertEvent(ctx, e); err != nil { + t.Fatal(err) + } + } + + rows, err := s.db.QueryContext(ctx, `SELECT at FROM events ORDER BY id`) + if err != nil { + t.Fatal(err) + } + defer func() { _ = rows.Close() }() + var ats []string + for rows.Next() { + var at string + if err := rows.Scan(&at); err != nil { + t.Fatal(err) + } + ats = append(ats, at) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + if len(ats) != 2 { + t.Fatalf("stored %d event rows, want 2", len(ats)) + } + if len(ats[0]) != len(ats[1]) { + t.Fatalf("at renders %d and %d chars (%q vs %q); want a fixed-width key", len(ats[0]), len(ats[1]), ats[0], ats[1]) + } +} + +// TestEventsSincePagesIdenticalTimestampsWithoutSkipOrRepeat is the +// regression guard for the id tie-break: many events sharing one timestamp +// must still page exactly once each, in a stable order, no matter what their +// ids are. It walks the returned cursor to exhaustion with a page size smaller +// than the set, so every page boundary is exercised. +func TestEventsSincePagesIdenticalTimestampsWithoutSkipOrRepeat(t *testing.T) { + s := openTest(t) + ctx := t.Context() + at := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + const total = 50 + inserted := make(map[int64]string, total) + for i := 0; i < total; i++ { + detail := fmt.Sprintf("event-%d", i) + id, err := s.InsertEvent(ctx, events.Event{Kind: "log", At: at, Detail: detail}) + if err != nil { + t.Fatal(err) + } + inserted[id] = detail + } + + var got []events.Event + cursor := "" + for { + page, err := s.EventsSince(ctx, cursor, 7) + if err != nil { + t.Fatal(err) + } + if len(page) == 0 { + break + } + got = append(got, page...) + last := page[len(page)-1] + cursor = storecontract.EventCursor(last.At, last.ID) + } + + if len(got) != total { + t.Fatalf("paged %d events, want %d (a skip or a repeat)", len(got), total) + } + seen := make(map[int64]bool, total) + for i, e := range got { + if _, ok := inserted[e.ID]; !ok { + t.Fatalf("event %d (%d) was never inserted", i, e.ID) + } + if seen[e.ID] { + t.Fatalf("event %d (%d) delivered twice", i, e.ID) + } + seen[e.ID] = true + if i > 0 { + prev := got[i-1] + if storecontract.EventCursor(prev.At, prev.ID) >= storecontract.EventCursor(e.At, e.ID) { + t.Fatalf("order regressed at %d: %q then %q", i, + storecontract.EventCursor(prev.At, prev.ID), storecontract.EventCursor(e.At, e.ID)) + } + } + } +} + // TestTasksBindingColumnsRoundTrip confirms the binding_id and // binding_version columns added in store schema 2 survive an INSERT and a // full SELECT round-trip through the Task read paths (scanTask, @@ -684,20 +810,24 @@ func TestEnqueueIsIdempotent(t *testing.T) { func TestEventLogRoundTrip(t *testing.T) { s := openTest(t) ctx := context.Background() + at := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) - id1, err := s.InsertEvent(ctx, events.Event{Kind: "stage_start", TaskID: 1, Stage: "plan", Attempt: 1}) + id1, err := s.InsertEvent(ctx, events.Event{Kind: "stage_start", TaskID: 1, Stage: "plan", Attempt: 1, At: at}) if err != nil || id1 == 0 { t.Fatalf("insert = (%d, %v)", id1, err) } id2, err := s.InsertEvent(ctx, events.Event{ - Kind: "stage_finish", TaskID: 1, Workflow: "implement", Stage: "plan", Attempt: 2, + Kind: "stage_finish", TaskID: 1, Workflow: "implement", Stage: "plan", Attempt: 2, At: at, Data: map[string]any{"duration_ms": 1200}, }) if err != nil || id2 <= id1 { t.Fatalf("second insert = (%d, %v)", id2, err) } - evs, err := s.EventsSince(ctx, id1, 10) + // Two events sharing a timestamp must still page exactly once each via the + // id tie-break: the cursor carries (at, id), so resuming after id1 yields + // only id2. + evs, err := s.EventsSince(ctx, storecontract.EventCursor(at, id1), 10) if err != nil || len(evs) != 1 || evs[0].Kind != "stage_finish" { t.Fatalf("EventsSince = (%+v, %v)", evs, err) } diff --git a/internal/webui/api_tasks_actions_test.go b/internal/webui/api_tasks_actions_test.go index cd27650a7..8e9977c14 100644 --- a/internal/webui/api_tasks_actions_test.go +++ b/internal/webui/api_tasks_actions_test.go @@ -686,7 +686,7 @@ func TestArchiveRemovesOnlyOneTerminalTask(t *testing.T) { if other, err := srv.Store.TaskByIssue(ctx, "acme", "widget", 2); err != nil || other == nil { t.Fatalf("archive removed another task: (%+v, %v)", other, err) } - history, err := srv.Store.EventsSince(ctx, 0, 10) + history, err := srv.Store.EventsSince(ctx, "", 10) if err != nil { t.Fatal(err) } diff --git a/internal/webui/event_pump.go b/internal/webui/event_pump.go index ce3d35600..6aba3e75f 100644 --- a/internal/webui/event_pump.go +++ b/internal/webui/event_pump.go @@ -6,6 +6,7 @@ import ( "fmt" "time" + "github.com/samcharles93/archie-core/internal/domain/storecontract" "github.com/samcharles93/archie-core/internal/events" ) @@ -38,18 +39,19 @@ const ( // // Polling is sufficient rather than a compromise, for the reasons recorded in // docs/architecture/migration-decisions.md ("Dashboard live event delivery"): -// the store serialises writes on a single connection, so the id cursor is -// assigned in commit order and cannot skip; sseStream.drain treats a broadcast -// as a wakeup and re-reads the gap from the store, so nothing here changes SSE -// semantics; and reading the table catches writers that never touch a bus, -// including the audit event ArchiveTask writes inside its transaction. +// EventsSince pages a total order over (at, id) with a fixed-width time key +// and a deterministic id tie-break, so the cursor cannot skip or repeat; +// sseStream.drain treats a broadcast as a wakeup and re-reads the gap from the +// store, so nothing here changes SSE semantics; and reading the table catches +// writers that never touch a bus, including the audit event ArchiveTask writes +// inside its transaction. type eventPump struct { store eventReader log func(string, ...any) broadcast func(events.Event) - // watermark is the highest event id already handed to broadcast. It only + // watermark is the highest event cursor already handed to broadcast. It only // ever moves forward, and only past an event that was delivered. - watermark int64 + watermark string // primeRetryMin is the first backoff between priming attempts. Zero // means eventPumpPrimeRetryMin; a test sets it small. primeRetryMin time.Duration @@ -85,7 +87,7 @@ func (p *eventPump) primeWithRetry(ctx context.Context) error { // eventReader is the one method the pump needs, so a test can drive it // without a whole TaskStore. type eventReader interface { - EventsSince(ctx context.Context, sinceID int64, limit int) ([]events.Event, error) + EventsSince(ctx context.Context, cursor string, limit int) ([]events.Event, error) } func (s *Server) newEventPump() *eventPump { @@ -122,8 +124,8 @@ func (p *eventPump) prime(ctx context.Context) error { } before := p.watermark for _, e := range batch { - if e.ID > p.watermark { - p.watermark = e.ID + if cursor := storecontract.EventCursor(e.At, e.ID); cursor > p.watermark { + p.watermark = cursor } } // A short page is the end of the table. The watermark check is the @@ -147,11 +149,12 @@ func (p *eventPump) deliver(ctx context.Context) (int, error) { } before := p.watermark for _, e := range batch { - if e.ID <= p.watermark { + cursor := storecontract.EventCursor(e.At, e.ID) + if cursor <= p.watermark { continue } p.broadcast(e) - p.watermark = e.ID + p.watermark = cursor delivered++ } if len(batch) < eventPumpPageSize || p.watermark == before { diff --git a/internal/webui/event_pump_test.go b/internal/webui/event_pump_test.go index 29f2d4712..c2f953073 100644 --- a/internal/webui/event_pump_test.go +++ b/internal/webui/event_pump_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/samcharles93/archie-core/internal/domain/storecontract" "github.com/samcharles93/archie-core/internal/events" ) @@ -26,8 +27,9 @@ func newTestPump(t *testing.T, srv *Server) *eventPump { // (sseStream.catchUp), not the pump's job. func TestEventPumpPrimesPastHistoryThenDeliversNewEvents(t *testing.T) { srv := newRemoteTestServer(t) + t0 := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) for _, detail := range []string{"first", "second"} { - if _, err := srv.Store.InsertEvent(t.Context(), events.Event{Kind: "task_created", Detail: detail}); err != nil { + if _, err := srv.Store.InsertEvent(t.Context(), events.Event{Kind: "task_created", Detail: detail, At: t0}); err != nil { t.Fatalf("seed history: %v", err) } } @@ -45,7 +47,8 @@ func TestEventPumpPrimesPastHistoryThenDeliversNewEvents(t *testing.T) { t.Fatalf("pump delivered %d pre-existing events; priming must advance past history", delivered) } - id, err := srv.Store.InsertEvent(t.Context(), events.Event{Kind: "task_started", Detail: "after the pump"}) + t1 := time.Date(2026, 1, 1, 0, 0, 1, 0, time.UTC) + id, err := srv.Store.InsertEvent(t.Context(), events.Event{Kind: "task_started", Detail: "after the pump", At: t1}) if err != nil { t.Fatalf("insert live event: %v", err) } @@ -56,8 +59,8 @@ func TestEventPumpPrimesPastHistoryThenDeliversNewEvents(t *testing.T) { if delivered != 1 { t.Fatalf("pump delivered %d events, want the 1 inserted after priming", delivered) } - if pump.watermark != id { - t.Fatalf("watermark = %d, want %d: the pump must not refetch a delivered event", pump.watermark, id) + if want := storecontract.EventCursor(t1, id); pump.watermark != want { + t.Fatalf("watermark = %q, want %q: the pump must not refetch a delivered event", pump.watermark, want) } } @@ -77,7 +80,7 @@ func TestEventPumpKeepsItsWatermarkAcrossAStoreFailure(t *testing.T) { t.Fatal("deliver against a cancelled context returned no error") } if pump.watermark != before { - t.Fatalf("watermark moved to %d on a failed fetch, want %d", pump.watermark, before) + t.Fatalf("watermark moved to %q on a failed fetch, want %q", pump.watermark, before) } } @@ -172,7 +175,7 @@ type flakyEventReader struct { calls int } -func (f *flakyEventReader) EventsSince(context.Context, int64, int) ([]events.Event, error) { +func (f *flakyEventReader) EventsSince(context.Context, string, int) ([]events.Event, error) { f.calls++ if f.calls <= f.failures { return nil, errors.New("connection refused") diff --git a/internal/webui/sse.go b/internal/webui/sse.go index 5acc24388..36729a06f 100644 --- a/internal/webui/sse.go +++ b/internal/webui/sse.go @@ -5,7 +5,6 @@ import ( "encoding/json" "log/slog" "net/http" - "strconv" "github.com/samcharles93/archie-core/internal/domain/storecontract" "github.com/samcharles93/archie-core/internal/events" @@ -46,22 +45,26 @@ func (s *Server) handleSSE(w http.ResponseWriter, r *http.Request) { conn, unregister := s.registerSSEConn() defer unregister() - if !stream.catchUp(r.Context(), 0) { + if !stream.catchUp(r.Context(), "") { return } stream.drain(r.Context(), conn) } -// sseSince resolves the event ID a client wants to resume after. A valid +// sseSince resolves the cursor a client wants to resume after. A valid // Last-Event-ID header -- how EventSource itself resumes a dropped // connection -- takes precedence over the ?since= query parameter, which -// only matters for a client's very first connection. -func sseSince(r *http.Request) int64 { - since, _ := strconv.ParseInt(r.URL.Query().Get("since"), 10, 64) +// only matters for a client's very first connection. An absent or unparseable +// cursor degrades to "from the beginning": a browser holding a legacy integer +// id (the pre-cursor id: field) or garbage must not break the feed, exactly +// as a failed ParseInt yielded 0 before the cursor became a string. +func sseSince(r *http.Request) string { + since := r.URL.Query().Get("since") if header := r.Header.Get("Last-Event-ID"); header != "" { - if fromHeader, err := strconv.ParseInt(header, 10, 64); err == nil { - since = fromHeader - } + since = header + } + if _, _, ok := storecontract.ParseEventCursor(since); !ok { + return "" } return since } @@ -86,19 +89,19 @@ func (s *Server) registerSSEConn() (chan events.Event, func()) { // sseStream renders one client's event-stream response: an SSE frame per // event, an SSE comment when the persisted backlog can't be read, and the -// running "since" watermark that deduplicates the backlog against live +// running "since" cursor that deduplicates the backlog against live // broadcasts covering the same event. type sseStream struct { store storecontract.TaskStore log *slog.Logger w http.ResponseWriter fl http.Flusher - since int64 + since string } // newSSEStream builds a stream over w, reporting false if w cannot be // flushed incrementally -- SSE does not work without that. -func newSSEStream(st storecontract.TaskStore, log *slog.Logger, w http.ResponseWriter, since int64) (*sseStream, bool) { +func newSSEStream(st storecontract.TaskStore, log *slog.Logger, w http.ResponseWriter, since string) (*sseStream, bool) { fl, ok := w.(http.Flusher) if !ok { return nil, false @@ -116,7 +119,7 @@ func (s *sseStream) send(e events.Event) bool { s.log.Error("sse event marshal failed", "error", err, "event_id", e.ID) return false } - if _, err := s.w.Write([]byte("id: " + strconv.FormatInt(e.ID, 10) + "\ndata: " + string(body) + "\n\n")); err != nil { + if _, err := s.w.Write([]byte("id: " + storecontract.EventCursor(e.At, e.ID) + "\ndata: " + string(body) + "\n\n")); err != nil { return false } s.fl.Flush() @@ -134,10 +137,10 @@ func (s *sseStream) writeErrorComment(err error) { } // catchUp sends every persisted event after s.since, advancing s.since as -// it goes, until it reaches targetID (0 means "drain everything currently +// it goes, until it reaches target ("" means "drain everything currently // available"), runs out of backlog, or fails. It reports false on failure // -- a fetch error or a dead connection -- meaning the caller must stop. -func (s *sseStream) catchUp(ctx context.Context, targetID int64) bool { +func (s *sseStream) catchUp(ctx context.Context, target string) bool { for { before := s.since backlog, err := s.store.EventsSince(ctx, s.since, sseBacklogPageSize) @@ -146,7 +149,7 @@ func (s *sseStream) catchUp(ctx context.Context, targetID int64) bool { s.writeErrorComment(err) return false } - reachedTarget, ok := s.sendPage(backlog, targetID) + reachedTarget, ok := s.sendPage(backlog, target) if !ok { return false } @@ -157,24 +160,25 @@ func (s *sseStream) catchUp(ctx context.Context, targetID int64) bool { } // sendPage sends every event in backlog newer than s.since, advancing it as -// it goes. reachedTarget reports whether targetID was reached mid-page -- +// it goes. reachedTarget reports whether target was reached mid-page -- // stopping there is correct, since nothing beyond it is needed yet -- and // ok reports whether every send succeeded. // // since advances past a filtered-out event (sseVisible false) exactly as it -// would for a sent one: EventsSince only ever returns events with -// id > since, so leaving since behind a filtered event would make the next -// catchUp refetch it forever. -func (s *sseStream) sendPage(backlog []events.Event, targetID int64) (reachedTarget, ok bool) { +// would for a sent one: EventsSince only ever returns events after since, so +// leaving since behind a filtered event would make the next catchUp refetch +// it forever. +func (s *sseStream) sendPage(backlog []events.Event, target string) (reachedTarget, ok bool) { for _, e := range backlog { - if e.ID <= s.since { + cursor := storecontract.EventCursor(e.At, e.ID) + if cursor <= s.since { continue } if sseVisible(e) && !s.send(e) { return false, false } - s.since = e.ID - if targetID > 0 && s.since >= targetID { + s.since = cursor + if target != "" && cursor >= target { return true, true } } @@ -192,17 +196,18 @@ func (s *sseStream) drain(ctx context.Context, conn <-chan events.Event) { case <-ctx.Done(): return case e := <-conn: - if e.ID <= s.since { + cursor := storecontract.EventCursor(e.At, e.ID) + if cursor <= s.since { continue } - if !s.catchUp(ctx, e.ID) { + if !s.catchUp(ctx, cursor) { return } - if e.ID > s.since { + if cursor > s.since { if sseVisible(e) && !s.send(e) { return } - s.since = e.ID + s.since = cursor } } } diff --git a/internal/webui/sse_test.go b/internal/webui/sse_test.go index de7ac3413..dd955ac07 100644 --- a/internal/webui/sse_test.go +++ b/internal/webui/sse_test.go @@ -8,12 +8,24 @@ import ( "net/http/httptest" "strings" "testing" + "time" + "github.com/samcharles93/archie-core/internal/domain/storecontract" "github.com/samcharles93/archie-core/internal/events" "github.com/samcharles93/archie-core/internal/store" ) -func newTestSSEStream(since int64) (*sseStream, *httptest.ResponseRecorder) { +// testAt is a fixed instant every synthetic event in this file shares, so a +// cursor can be built for an event from its id alone. Two events sharing a +// timestamp exercise the id tie-break, which is the point of the cursor. +var testAt = time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + +// cursor builds the opaque resume cursor for a synthetic event with id id. +func cursor(id int64) string { + return storecontract.EventCursor(testAt, id) +} + +func newTestSSEStream(since string) (*sseStream, *httptest.ResponseRecorder) { rec := httptest.NewRecorder() return &sseStream{ log: slog.New(slog.DiscardHandler), @@ -24,13 +36,19 @@ func newTestSSEStream(since int64) (*sseStream, *httptest.ResponseRecorder) { } // sendPage is the seam catchUp's page-boundary logic depends on: it must -// stop the instant targetID is reached, even mid-page, rather than sending +// stop the instant target is reached, even mid-page, rather than sending // events beyond what the caller asked for. func TestSSEStreamSendPageStopsAtTargetMidPage(t *testing.T) { - s, _ := newTestSSEStream(0) - backlog := []events.Event{{ID: 1}, {ID: 2}, {ID: 3}, {ID: 4}, {ID: 5}} + s, _ := newTestSSEStream("") + backlog := []events.Event{ + {ID: 1, At: testAt}, + {ID: 2, At: testAt}, + {ID: 3, At: testAt}, + {ID: 4, At: testAt}, + {ID: 5, At: testAt}, + } - reachedTarget, ok := s.sendPage(backlog, 3) + reachedTarget, ok := s.sendPage(backlog, cursor(3)) if !ok { t.Fatal("sendPage() ok = false, want true") @@ -38,18 +56,18 @@ func TestSSEStreamSendPageStopsAtTargetMidPage(t *testing.T) { if !reachedTarget { t.Fatal("reachedTarget = false, want true: target 3 was inside the page") } - if s.since != 3 { - t.Fatalf("since = %d, want 3 (stopped at the target, not the end of the page)", s.since) + if s.since != cursor(3) { + t.Fatalf("since = %q, want %q (stopped at the target, not the end of the page)", s.since, cursor(3)) } } // A target beyond the page must not be reported reached: the caller needs // another page. func TestSSEStreamSendPageDoesNotReportTargetReachedBeyondThePage(t *testing.T) { - s, _ := newTestSSEStream(0) - backlog := []events.Event{{ID: 1}, {ID: 2}, {ID: 3}} + s, _ := newTestSSEStream("") + backlog := []events.Event{{ID: 1, At: testAt}, {ID: 2, At: testAt}, {ID: 3, At: testAt}} - reachedTarget, ok := s.sendPage(backlog, 10) + reachedTarget, ok := s.sendPage(backlog, cursor(10)) if !ok { t.Fatal("sendPage() ok = false, want true") @@ -57,36 +75,36 @@ func TestSSEStreamSendPageDoesNotReportTargetReachedBeyondThePage(t *testing.T) if reachedTarget { t.Fatal("reachedTarget = true, want false: target 10 was never reached in this page") } - if s.since != 3 { - t.Fatalf("since = %d, want 3 (the whole page was sent)", s.since) + if s.since != cursor(3) { + t.Fatalf("since = %q, want %q (the whole page was sent)", s.since, cursor(3)) } } -// targetID == 0 means "drain everything available"; sendPage must never +// target == "" means "drain everything available"; sendPage must never // treat it as a target to stop at, or catchUp's initial call (which always -// passes 0) would stop after the very first event. +// passes "") would stop after the very first event. func TestSSEStreamSendPageIgnoresZeroTarget(t *testing.T) { - s, _ := newTestSSEStream(0) - backlog := []events.Event{{ID: 1}, {ID: 2}, {ID: 3}} + s, _ := newTestSSEStream("") + backlog := []events.Event{{ID: 1, At: testAt}, {ID: 2, At: testAt}, {ID: 3, At: testAt}} - reachedTarget, ok := s.sendPage(backlog, 0) + reachedTarget, ok := s.sendPage(backlog, "") if !ok || reachedTarget { - t.Fatalf("sendPage(targetID=0) = (%v, %v), want (false, true)", reachedTarget, ok) + t.Fatalf("sendPage(target=\"\") = (%v, %v), want (false, true)", reachedTarget, ok) } - if s.since != 3 { - t.Fatalf("since = %d, want 3 (the whole page was sent)", s.since) + if s.since != cursor(3) { + t.Fatalf("since = %q, want %q (the whole page was sent)", s.since, cursor(3)) } } // A send failure must stop the page immediately and report failure, not // silently skip the failed event and keep going. func TestSSEStreamSendPageStopsOnWriteFailure(t *testing.T) { - s, _ := newTestSSEStream(0) + s, _ := newTestSSEStream("") s.w = failingWriter{} - backlog := []events.Event{{ID: 1}, {ID: 2}} + backlog := []events.Event{{ID: 1, At: testAt}, {ID: 2, At: testAt}} - reachedTarget, ok := s.sendPage(backlog, 0) + reachedTarget, ok := s.sendPage(backlog, "") if ok { t.Fatal("sendPage() ok = true, want false: the write failed") @@ -94,8 +112,8 @@ func TestSSEStreamSendPageStopsOnWriteFailure(t *testing.T) { if reachedTarget { t.Fatal("reachedTarget = true, want false on failure") } - if s.since != 0 { - t.Fatalf("since = %d, want 0 (unchanged: the first send already failed)", s.since) + if s.since != "" { + t.Fatalf("since = %q, want empty (unchanged: the first send already failed)", s.since) } } @@ -110,16 +128,16 @@ func (failingWriter) WriteHeader(statusCode int) {} var errWriteFailed = errors.New("write failed") // fakeEventStore serves EventsSince from a fixed, pre-seeded slice, paginating -// the same way the real store does: up to limit events with id > sinceID. +// the same way the real store does: up to limit events after cursor. type fakeEventStore struct { store.TaskStore all []events.Event } -func (f *fakeEventStore) EventsSince(_ context.Context, sinceID int64, limit int) ([]events.Event, error) { +func (f *fakeEventStore) EventsSince(_ context.Context, since string, limit int) ([]events.Event, error) { page := make([]events.Event, 0, limit) for _, e := range f.all { - if e.ID <= sinceID { + if storecontract.EventCursor(e.At, e.ID) <= since { continue } page = append(page, e) @@ -137,40 +155,40 @@ func TestSSEStreamCatchUpPagesAcrossMultipleFetches(t *testing.T) { const total = sseBacklogPageSize + 50 all := make([]events.Event, total) for i := range all { - all[i] = events.Event{ID: int64(i + 1)} + all[i] = events.Event{ID: int64(i + 1), At: testAt} } - s, rec := newTestSSEStream(0) + s, rec := newTestSSEStream("") s.store = &fakeEventStore{all: all} - if !s.catchUp(context.Background(), 0) { + if !s.catchUp(context.Background(), "") { t.Fatal("catchUp() = false, want true") } - if s.since != int64(total) { - t.Fatalf("since = %d, want %d (every event drained across pages)", s.since, total) + if s.since != cursor(total) { + t.Fatalf("since = %q, want %q (every event drained across pages)", s.since, cursor(total)) } if n := countSSEFrames(rec.Body.String()); n != total { t.Fatalf("sent %d SSE frames, want %d", n, total) } } -// A targetID landing exactly on the last event of a full page must still be +// A target landing exactly on the last event of a full page must still be // recognised as reached, not mistaken for "page not yet exhausted, fetch // another". func TestSSEStreamCatchUpStopsAtTargetOnPageBoundary(t *testing.T) { all := make([]events.Event, sseBacklogPageSize+50) for i := range all { - all[i] = events.Event{ID: int64(i + 1)} + all[i] = events.Event{ID: int64(i + 1), At: testAt} } - s, _ := newTestSSEStream(0) + s, _ := newTestSSEStream("") s.store = &fakeEventStore{all: all} - if !s.catchUp(context.Background(), sseBacklogPageSize) { + if !s.catchUp(context.Background(), cursor(sseBacklogPageSize)) { t.Fatal("catchUp() = false, want true") } - if s.since != sseBacklogPageSize { - t.Fatalf("since = %d, want %d (stopped exactly at the target, not the whole backlog)", s.since, sseBacklogPageSize) + if s.since != cursor(sseBacklogPageSize) { + t.Fatalf("since = %q, want %q (stopped exactly at the target, not the whole backlog)", s.since, cursor(sseBacklogPageSize)) } } @@ -186,23 +204,23 @@ func countSSEFrames(body string) int { // the dashboard's Live Activity panel. It must never reach the SSE client, // but since must still advance past it -- otherwise the next catchUp // refetches the same filtered event forever, since EventsSince only ever -// returns events with id > since. +// returns events after since. func TestSSEStreamSendPageSkipsNoiseKindsButAdvancesSince(t *testing.T) { - s, rec := newTestSSEStream(0) + s, rec := newTestSSEStream("") backlog := []events.Event{ - {ID: 1, Kind: events.KindTaskQueued}, - {ID: 2, Kind: events.KindTurnCompleted, Detail: "100000000"}, - {ID: 3, Kind: events.KindTurnCompleted, Detail: "100000000"}, - {ID: 4, Kind: events.KindParked}, + {ID: 1, At: testAt, Kind: events.KindTaskQueued}, + {ID: 2, At: testAt, Kind: events.KindTurnCompleted, Detail: "100000000"}, + {ID: 3, At: testAt, Kind: events.KindTurnCompleted, Detail: "100000000"}, + {ID: 4, At: testAt, Kind: events.KindParked}, } - reachedTarget, ok := s.sendPage(backlog, 0) + reachedTarget, ok := s.sendPage(backlog, "") if !ok || reachedTarget { t.Fatalf("sendPage() = (%v, %v), want (false, true)", reachedTarget, ok) } - if s.since != 4 { - t.Fatalf("since = %d, want 4 (advanced past every event, including filtered ones)", s.since) + if s.since != cursor(4) { + t.Fatalf("since = %q, want %q (advanced past every event, including filtered ones)", s.since, cursor(4)) } if n := countSSEFrames(rec.Body.String()); n != 2 { t.Fatalf("sent %d SSE frames, want 2 (turn_completed excluded)", n) @@ -211,3 +229,41 @@ func TestSSEStreamSendPageSkipsNoiseKindsButAdvancesSince(t *testing.T) { t.Error("SSE body contains a turn_completed frame, want it excluded entirely") } } + +// TestSSESinceDegradesUnparseableCursorToBeginning is the legacy-cursor +// tolerance guard: a browser holding the pre-cursor integer id: field, or a +// ?since= carrying garbage, must resume from the beginning rather than error +// or skip. This deliberately preserves the old failed-ParseInt-yields-0 +// degradation. +func TestSSESinceDegradesUnparseableCursorToBeginning(t *testing.T) { + valid := storecontract.EventCursor(testAt, 1) + tests := []struct { + name string + since string + header string + want string + }{ + {name: "absent", want: ""}, + {name: "legacy integer header", header: "42", want: ""}, + {name: "garbage query", since: "not-a-cursor", want: ""}, + {name: "garbage header", header: "garbage", want: ""}, + {name: "valid query cursor", since: valid, want: valid}, + {name: "valid header cursor beats garbage query", since: "garbage", header: valid, want: valid}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/events", nil) + if tt.since != "" { + q := req.URL.Query() + q.Set("since", tt.since) + req.URL.RawQuery = q.Encode() + } + if tt.header != "" { + req.Header.Set("Last-Event-ID", tt.header) + } + if got := sseSince(req); got != tt.want { + t.Fatalf("sseSince = %q, want %q", got, tt.want) + } + }) + } +} diff --git a/internal/webui/webui_test.go b/internal/webui/webui_test.go index 5b347cfb6..5b70d4c7f 100644 --- a/internal/webui/webui_test.go +++ b/internal/webui/webui_test.go @@ -26,6 +26,7 @@ import ( "testing" "time" + "github.com/samcharles93/archie-core/internal/domain/storecontract" "github.com/samcharles93/archie-core/internal/domain/workflow" "github.com/samcharles93/archie-core/internal/events" "github.com/samcharles93/archie-core/internal/store" @@ -95,11 +96,11 @@ func (s *stubStore) TokensByDay(ctx context.Context, days int) ([]store.DayToken return s.TaskStore.TokensByDay(ctx, days) } -func (s *stubStore) EventsSince(ctx context.Context, sinceID int64, limit int) ([]events.Event, error) { +func (s *stubStore) EventsSince(ctx context.Context, cursor string, limit int) ([]events.Event, error) { if s.eventsSinceErr != nil { return nil, s.eventsSinceErr } - return s.TaskStore.EventsSince(ctx, sinceID, limit) + return s.TaskStore.EventsSince(ctx, cursor, limit) } func (s *stubStore) TaskEvents(ctx context.Context, taskID int64) ([]events.Event, error) { @@ -350,11 +351,12 @@ func TestHandleSSEUsesLastEventIDForCatchUp(t *testing.T) { srv := newTestServer(t) ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) defer cancel() - first, err := srv.Store.InsertEvent(ctx, events.Event{Kind: "log", Detail: "first"}) + at := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + first, err := srv.Store.InsertEvent(ctx, events.Event{Kind: "log", Detail: "first", At: at}) if err != nil { t.Fatal(err) } - if _, err := srv.Store.InsertEvent(ctx, events.Event{Kind: "log", Detail: "second"}); err != nil { + if _, err := srv.Store.InsertEvent(ctx, events.Event{Kind: "log", Detail: "second", At: at}); err != nil { t.Fatal(err) } @@ -364,7 +366,7 @@ func TestHandleSSEUsesLastEventIDForCatchUp(t *testing.T) { if err != nil { t.Fatal(err) } - req.Header.Set("Last-Event-ID", strconv.FormatInt(first, 10)) + req.Header.Set("Last-Event-ID", storecontract.EventCursor(at, first)) resp, err := http.DefaultClient.Do(req) if err != nil { t.Fatal(err) @@ -394,7 +396,8 @@ func TestHandleSSEBacklogAndLive(t *testing.T) { srv := newTestServer(t) ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) defer cancel() - id, err := srv.Store.InsertEvent(ctx, events.Event{Kind: "log", Detail: "backlog"}) + t0 := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + id, err := srv.Store.InsertEvent(ctx, events.Event{Kind: "log", Detail: "backlog", At: t0}) if err != nil { t.Fatal(err) } @@ -426,7 +429,7 @@ func TestHandleSSEBacklogAndLive(t *testing.T) { // Now broadcast a live event and confirm it's delivered too. Give // the SSE handler a moment to register its subscriber. time.Sleep(50 * time.Millisecond) - srv.Broadcast(events.Event{ID: id + 1, Kind: "log", Detail: "live"}) + srv.Broadcast(events.Event{ID: id + 1, Kind: "log", Detail: "live", At: t0.Add(time.Second)}) if _, err := readLineUntil(reader, "live"); err != nil { t.Fatalf("live event not received: %v", err) @@ -442,7 +445,8 @@ func TestHandleSSEFiltersTurnCompletedFromLiveBroadcast(t *testing.T) { srv := newTestServer(t) ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) defer cancel() - id, err := srv.Store.InsertEvent(ctx, events.Event{Kind: "log", Detail: "backlog"}) + t0 := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + id, err := srv.Store.InsertEvent(ctx, events.Event{Kind: "log", Detail: "backlog", At: t0}) if err != nil { t.Fatal(err) } @@ -469,8 +473,8 @@ func TestHandleSSEFiltersTurnCompletedFromLiveBroadcast(t *testing.T) { } time.Sleep(50 * time.Millisecond) - srv.Broadcast(events.Event{ID: id + 1, Kind: events.KindTurnCompleted, Detail: "100000000"}) - srv.Broadcast(events.Event{ID: id + 2, Kind: "log", Detail: "after-noise"}) + srv.Broadcast(events.Event{ID: id + 1, Kind: events.KindTurnCompleted, Detail: "100000000", At: t0.Add(time.Second)}) + srv.Broadcast(events.Event{ID: id + 2, Kind: "log", Detail: "after-noise", At: t0.Add(2 * time.Second)}) seen, err := readLinesUntil(reader, "after-noise") if err != nil { diff --git a/proto/state/v1/state.proto b/proto/state/v1/state.proto index c24340147..7778ca162 100644 --- a/proto/state/v1/state.proto +++ b/proto/state/v1/state.proto @@ -470,7 +470,10 @@ message InsertEventRequest { Event event = 1; } message InsertEventResponse { int64 id = 1; } message EventsSinceRequest { - int64 since_id = 1; + // cursor is the opaque resume cursor returned by the previous page (the SSE + // id: field, echoed back as Last-Event-ID). Empty means "from the + // beginning". + string cursor = 1; int64 limit = 2; } message EventsSinceResponse { repeated Event events = 1; } From c208a637cbe790399cf83e3d558f68df13021d1c Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Wed, 23 Sep 2026 03:12:48 +1000 Subject: [PATCH 15/20] fix(proto): keep EventsSinceRequest.since_id's wire name The cursor change renamed the field to 'cursor' as well as widening its type, which buf breaking flags as FIELD_SAME_NAME and FIELD_SAME_JSON_NAME on top of the type change. The name buys nothing: the field is already documented as an opaque cursor, and the number and name are the wire identity. Adopt the gate's verdict and change only the type, keeping the break confined to the one rule already recorded in proto/buf.yaml. --- internal/contracts/state/v1/state.pb.go | 18 ++++++++++-------- internal/infrastructure/staterpc/client.go | 2 +- internal/infrastructure/staterpc/server.go | 2 +- proto/buf.yaml | 21 ++++++++++++--------- proto/state/v1/state.proto | 8 +++++--- 5 files changed, 29 insertions(+), 22 deletions(-) diff --git a/internal/contracts/state/v1/state.pb.go b/internal/contracts/state/v1/state.pb.go index cacdc3308..8a2d725d9 100644 --- a/internal/contracts/state/v1/state.pb.go +++ b/internal/contracts/state/v1/state.pb.go @@ -4134,10 +4134,12 @@ func (x *InsertEventResponse) GetId() int64 { type EventsSinceRequest struct { state protoimpl.MessageState `protogen:"open.v1"` - // cursor is the opaque resume cursor returned by the previous page (the SSE + // since_id is the opaque resume cursor returned by the previous page (the SSE // id: field, echoed back as Last-Event-ID). Empty means "from the - // beginning". - Cursor string `protobuf:"bytes,1,opt,name=cursor,proto3" json:"cursor,omitempty"` + // beginning". The field keeps its name and number so the wire name stays + // put; only its type widened, from the SQLite rowid to the fixed-width + // "|" cursor. + SinceId string `protobuf:"bytes,1,opt,name=since_id,json=sinceId,proto3" json:"since_id,omitempty"` Limit int64 `protobuf:"varint,2,opt,name=limit,proto3" json:"limit,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -4173,9 +4175,9 @@ func (*EventsSinceRequest) Descriptor() ([]byte, []int) { return file_state_v1_state_proto_rawDescGZIP(), []int{70} } -func (x *EventsSinceRequest) GetCursor() string { +func (x *EventsSinceRequest) GetSinceId() string { if x != nil { - return x.Cursor + return x.SinceId } return "" } @@ -7928,9 +7930,9 @@ const file_state_v1_state_proto_rawDesc = "" + "\x12InsertEventRequest\x12%\n" + "\x05event\x18\x01 \x01(\v2\x0f.state.v1.EventR\x05event\"%\n" + "\x13InsertEventResponse\x12\x0e\n" + - "\x02id\x18\x01 \x01(\x03R\x02id\"B\n" + - "\x12EventsSinceRequest\x12\x16\n" + - "\x06cursor\x18\x01 \x01(\tR\x06cursor\x12\x14\n" + + "\x02id\x18\x01 \x01(\x03R\x02id\"E\n" + + "\x12EventsSinceRequest\x12\x19\n" + + "\bsince_id\x18\x01 \x01(\tR\asinceId\x12\x14\n" + "\x05limit\x18\x02 \x01(\x03R\x05limit\">\n" + "\x13EventsSinceResponse\x12'\n" + "\x06events\x18\x01 \x03(\v2\x0f.state.v1.EventR\x06events\",\n" + diff --git a/internal/infrastructure/staterpc/client.go b/internal/infrastructure/staterpc/client.go index 9b18c5a7e..2bff29d64 100644 --- a/internal/infrastructure/staterpc/client.go +++ b/internal/infrastructure/staterpc/client.go @@ -216,7 +216,7 @@ func (c *Client) InsertEvent(ctx context.Context, e events.Event) (int64, error) } func (c *Client) EventsSince(ctx context.Context, cursor string, limit int) ([]events.Event, error) { - r, err := c.client.EventsSince(ctx, &pb.EventsSinceRequest{Cursor: cursor, Limit: int64(limit)}) + r, err := c.client.EventsSince(ctx, &pb.EventsSinceRequest{SinceId: cursor, Limit: int64(limit)}) if err != nil { return nil, unmapError(err) } diff --git a/internal/infrastructure/staterpc/server.go b/internal/infrastructure/staterpc/server.go index 11097ef05..68b93a3c3 100644 --- a/internal/infrastructure/staterpc/server.go +++ b/internal/infrastructure/staterpc/server.go @@ -283,7 +283,7 @@ func (s *server) InsertEvent(ctx context.Context, r *pb.InsertEventRequest) (*pb } func (s *server) EventsSince(ctx context.Context, r *pb.EventsSinceRequest) (*pb.EventsSinceResponse, error) { - evs, err := s.deps.Tasks.EventsSince(ctx, r.Cursor, int(r.Limit)) + evs, err := s.deps.Tasks.EventsSince(ctx, r.SinceId, int(r.Limit)) if err != nil { return nil, s.logErr("EventsSince", err) } diff --git a/proto/buf.yaml b/proto/buf.yaml index 6d9f7222d..17f3cea97 100644 --- a/proto/buf.yaml +++ b/proto/buf.yaml @@ -7,15 +7,18 @@ lint: breaking: use: - FILE - # The PocketBase migration moved the event-capture contract's record ids from - # the SQLite rowid (int64) to PocketBase's string record ids, in place, on the - # same field numbers: 16 fields across CapturedEvent, Mapping, Binding, - # RecordDispatchRequest, EnqueueBindingTaskRequest, Task.binding_id and the - # Id-bearing request/response messages. That is an intentional wire break, and - # both ends of the State Store contract ship from this repository together, so - # the two halves cannot be versioned apart. Only the type rule is relaxed, and - # only in this file: every other FILE breaking rule still applies here, and no - # other contract is affected. + # Two intentional wire breaks, both widening an id-shaped field from int64 to + # string in place, on the same field numbers and names. The PocketBase + # migration moved the event-capture contract's record ids to PocketBase's + # string record ids: CapturedEvent, Mapping, Binding, RecordDispatchRequest, + # EnqueueBindingTaskRequest, Task.binding_id and the Id-bearing + # request/response messages. Separately, EventsSinceRequest.since_id now + # carries the opaque "|" resume cursor instead of a rowid. + # + # The State Store contract's two halves ship from this repository together, so + # they cannot be versioned apart. Only the type rule is relaxed, and only in + # this file: every other FILE breaking rule still applies here, so a rename, + # a renumber or a removal is still caught, and no other contract is affected. ignore_only: FIELD_SAME_TYPE: - state/v1/state.proto diff --git a/proto/state/v1/state.proto b/proto/state/v1/state.proto index 7778ca162..69b40a9c4 100644 --- a/proto/state/v1/state.proto +++ b/proto/state/v1/state.proto @@ -470,10 +470,12 @@ message InsertEventRequest { Event event = 1; } message InsertEventResponse { int64 id = 1; } message EventsSinceRequest { - // cursor is the opaque resume cursor returned by the previous page (the SSE + // since_id is the opaque resume cursor returned by the previous page (the SSE // id: field, echoed back as Last-Event-ID). Empty means "from the - // beginning". - string cursor = 1; + // beginning". The field keeps its name and number so the wire name stays + // put; only its type widened, from the SQLite rowid to the fixed-width + // "|" cursor. + string since_id = 1; int64 limit = 2; } message EventsSinceResponse { repeated Event events = 1; } From e0becd525c914ce676b5be1962836a414af50967 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Wed, 23 Sep 2026 03:15:27 +1000 Subject: [PATCH 16/20] feat(artifacts): publish the completed PR review to the collaborative editor Implements docs/prds/artifact-sender.md (Approved). archie-core now sends generated artifacts to the offloaded.dev collaborative editor, whose POST /api/artifacts route, bearer service token, PocketBase-backed versions and idempotency semantics are the receiving contract. - [artifacts] config section: base_url + token SecretRef with a token_env fallback naming WORKSPACE_INGEST_TOKEN. An empty base_url disables the sender entirely; a base_url with an unresolvable token disables it with a warning, because a missing credential disables a capability rather than stopping the daemon. - internal/infrastructure/artifactsync: one-method Publish seam in the Courier posture, full source per publish (versions of record are snapshots), 503 retried three times with backoff, 401/400 surfaced immediately with the field reason, deterministic request ids (artifact:pr-review:/#). - ReviewReport gains a pure markdown renderer in internal/domain/workflow: summary, one subsection per finding with the failure scenario and disposition when present, verified properties as a checked-clean list. - The daemon's PR reviewer publishes after a completed review when the sender is configured, records the editor deep link on the result, and degrades a failed publish to a log line -- publishing never fails the producing work. --- config.example.toml | 8 + docs/prds/artifact-sender.md | 83 ++++++++ internal/app/archied/pr_review.go | 94 ++++++++- internal/app/archied/pr_review_test.go | 74 +++++++ internal/config/config.go | 18 ++ internal/domain/workflow/review_report_md.go | 56 ++++++ .../domain/workflow/review_report_md_test.go | 78 ++++++++ internal/gateway/review_tools.go | 3 + .../infrastructure/artifactsync/sender.go | 181 ++++++++++++++++++ .../artifactsync/sender_test.go | 159 +++++++++++++++ .../configuration/artifacts_config_test.go | 52 +++++ .../infrastructure/configuration/defaults.go | 12 ++ 12 files changed, 817 insertions(+), 1 deletion(-) create mode 100644 docs/prds/artifact-sender.md create mode 100644 internal/domain/workflow/review_report_md.go create mode 100644 internal/domain/workflow/review_report_md_test.go create mode 100644 internal/infrastructure/artifactsync/sender.go create mode 100644 internal/infrastructure/artifactsync/sender_test.go create mode 100644 internal/infrastructure/configuration/artifacts_config_test.go diff --git a/config.example.toml b/config.example.toml index bd7aa71d3..f20a4fbfd 100644 --- a/config.example.toml +++ b/config.example.toml @@ -227,6 +227,14 @@ class = "ollama" # work_dir = "/path/to/workspace" # parallel_tool_calls = false +# Publish artifacts (review reports, design docs) to the collaborative editor. +# An empty base_url disables publishing entirely. The token is runtime config: +# set WORKSPACE_INGEST_TOKEN in the daemon's environment or use a secret ref. +[artifacts] +# base_url = "https://offloaded.dev" +# token = { engine = "env", key = "WORKSPACE_INGEST_TOKEN" } +# token_env = "WORKSPACE_INGEST_TOKEN" + # archied's own liveness surface, always on. GET /healthz and /health answer # 200 once the daemon is serving (503 while it is still booting), and # /health/detailed reports the readiness probes. This is what the update diff --git a/docs/prds/artifact-sender.md b/docs/prds/artifact-sender.md new file mode 100644 index 000000000..64924072e --- /dev/null +++ b/docs/prds/artifact-sender.md @@ -0,0 +1,83 @@ +# Artifact sender + +**Status:** Approved +**Revision:** 1 +**Authority:** this document + +## Purpose + +archie-core publishes artifacts — design documents, review reports, research briefs, +runnable pages — to the offloaded.dev collaborative editor. A person reads the +artifact, comments on it, and the artifact page is the shared surface for iteration. + +The receiving contract is owned by the editor: `POST /api/artifacts` with a bearer +service token, a closed field set, and server-side idempotency on `requestId` plus +last-write-wins on `sentAt`. This document owns only what archie-core sends. + +## Scope + +Version one publishes one-way: archie-core sends, the editor receives. The first +producer is the pull-request adversarial review: a completed `ReviewReport` is +rendered to markdown and published after the review completes. + +Chat-surface tools (`create_artifact`, `update_artifact`), comment read-back, share +tokens, and runnable-format producers (`html`, `react`, `vue`) are later slices. The +receiving contract already carries them; version one does not exercise them. + +## Configuration + +- New `[artifacts]` section: `base_url` (site origin), `token` (`SecretRef`) with + `token_env` fallback naming `WORKSPACE_INGEST_TOKEN`. +- An empty `base_url` disables the sender: no client is constructed, nothing is + registered, and publishing degrades to a log line. +- The token is runtime configuration. It is never baked at build time. + +## Sender + +- New package `internal/infrastructure/artifactsync` exposing one method, + `Publish(ctx, Artifact) (string, error)`, where the string is the artifact URL. + One-method injected seam, concrete client constructed by the boot path when the + configuration resolves — the `crondelivery.Courier` posture. +- The client POSTs the frozen contract body. Each publish sends the complete + source: versions of record are full snapshots, never diffs. +- A `503` response is retried three times with backoff. `401` and `400` are not + retried; they are logged with the field reason the response names. +- `requestId` is `artifact::` when the producer is task-backed, + otherwise `artifact::`. Sending the same `requestId` with + an equal or older `sentAt` must be a server-side no-op, which makes sender-side + retry safe. + +## Report rendering + +- `ReviewReport` gains a pure markdown renderer in `internal/domain/workflow`: + the summary as the lede, one finding per subsection with file, line, verdict, + level and the failure scenario, and the verified properties as a "checked + clean" list. Findings carry their disposition when present. +- The renderer performs no I/O and imports no infrastructure. + +## Attribution + +- `actor`: the producing identity. The reviewer publishes as `system:pr-reviewer` + when no task identity resolves; otherwise the task's identity name. +- `source`: the producing subsystem, `pr-review` for version one. +- `task`: owner, repository and issue number from the task when one exists. + +## Failure behaviour + +- Publishing never fails the producing work. A failed publish is logged with its + status and reason; the artifact is republished on the next production of the + same artifact. +- A successful publish records the artifact URL in the run report, so the PR + description or task log links to the readable artifact. +- The published URL is the editor deep link for the artifact, keyed on its + immutable id. archie-core never mints share links; sharing is an + account-holder decision made in the editor. + +## Verification + +- Client tests against a test server cover 201, 401, 400 with a field reason, + and 503 retry behaviour. +- The report renderer test covers a report with findings, checks, dispositions + and a not-run status. +- Configuration tests cover the disabled case, the `SecretRef` path and the + `token_env` fallback. \ No newline at end of file diff --git a/internal/app/archied/pr_review.go b/internal/app/archied/pr_review.go index 55b2bbb30..30116537d 100644 --- a/internal/app/archied/pr_review.go +++ b/internal/app/archied/pr_review.go @@ -8,6 +8,7 @@ import ( "strconv" "strings" "sync" + "time" "github.com/samcharles93/archie-core/internal/agentexec" "github.com/samcharles93/archie-core/internal/app/agentworker" @@ -15,6 +16,7 @@ import ( "github.com/samcharles93/archie-core/internal/domain/workflow" "github.com/samcharles93/archie-core/internal/forge" "github.com/samcharles93/archie-core/internal/gateway" + "github.com/samcharles93/archie-core/internal/infrastructure/artifactsync" "github.com/samcharles93/archie-core/internal/worktree" ) @@ -29,6 +31,9 @@ type prReviewer struct { models map[string]string providers map[string]agentexec.Provider maxSteps int + // artifacts publishes the completed review to the collaborative editor. + // nil = publishing is disabled (no [artifacts] base_url configured). + artifacts artifactsync.Sender // currentMaxSteps lets the composition root apply live control-plane // versions to reviews that have not started yet. Tests and direct // constructions retain maxSteps as the fallback. @@ -115,7 +120,9 @@ func (r *prReviewer) ReviewPR(ctx context.Context, identity *string, repo string MaxSteps: maxSteps, }) r.logReview(key, model, report) - return mapReviewResult(repo, number, model, report), nil + + result := mapReviewResult(repo, number, model, report) + return r.attachArtifact(ctx, result, identity, owner, name, number, report), nil } // authorize enforces the identity-scoped repository allow-list. A nil @@ -174,6 +181,62 @@ func (r *prReviewer) logReview(key, model string, report workflow.ReviewReport) "findings", len(report.Findings), "blocking", blocking) } +// attachArtifact publishes the report when the sender is configured and +// records the editor deep link on the result. Publishing never fails the +// review: a failure degrades to an empty URL. +func (r *prReviewer) attachArtifact(ctx context.Context, result gateway.PRReviewResult, identity *string, owner, name string, number int, report workflow.ReviewReport) gateway.PRReviewResult { + result.ArtifactURL = r.publishArtifact(ctx, reviewActor(identity), owner, name, number, report) + return result +} + +// reviewActor resolves the attribution: the requesting chat identity when one +// exists, else the daemon's operator-reviewer actor. +func reviewActor(identity *string) string { + if identity != nil && *identity != "" { + return *identity + } + return "system:pr-reviewer" +} + +// publishArtifact renders the report and hands it to the collaborative +// editor when artifact publishing is configured. It returns the editor deep +// link, or "" when publishing is disabled or failed: a publish failure is a +// logged degradation, never an error that fails the producing review. +func (r *prReviewer) publishArtifact(ctx context.Context, actor, owner, name string, number int, report workflow.ReviewReport) string { + if r.artifacts == nil { + return "" + } + title := fmt.Sprintf("Review of %s/%s#%d", owner, name, number) + artifact := artifactsync.Artifact{ + Title: title, + Source: report.RenderMarkdown(title), + Meta: artifactsync.Meta{ + Format: "markdown", + Category: "pr-review", + Visibility: "account-only", + Egress: "none", + Actor: actor, + Origin: "pr-review", + RequestID: "artifact:pr-review:" + owner + "/" + name + "#" + strconv.Itoa(number), + Version: "r1", + SentAt: time.Now().UTC(), + Task: &artifactsync.Task{Owner: owner, Repo: name, Number: number}, + }, + } + url, err := r.artifacts.Publish(ctx, artifact) + if err != nil { + if r.log != nil { + r.log.Warn("artifact publish failed; the review result is unaffected", + "artifact", artifact.RequestID(), "err", err) + } + return "" + } + if r.log != nil { + r.log.Info("artifact published", "artifact", artifact.RequestID(), "url", url) + } + return url +} + // splitOwnerRepo splits "owner/name" with the same validation the gateway's // task_spawn allow-list uses. func splitOwnerRepo(s string) (owner, repo string, err error) { @@ -264,6 +327,7 @@ func (b *boot) prReviewer() gateway.ChatPRReviewer { models: b.cfg.Models, providers: executionProviders(b.cfg), maxSteps: maxSteps, + artifacts: b.artifactSender(), allowed: buildReviewAllowlist(b.cfg), log: b.log, } @@ -273,3 +337,31 @@ func (b *boot) prReviewer() gateway.ChatPRReviewer { b.chatPRReviewer = review return b.chatPRReviewer } + +// artifactSender builds the collaborative-editor sender when the [artifacts] +// section configures one. An empty base_url disables publishing entirely; a +// base_url with an unresolvable token is a missing credential, which disables +// the capability with a loud warning rather than stopping the daemon +// (docs/architecture/configuration.md, "A missing credential disables a +// capability"). +func (b *boot) artifactSender() artifactsync.Sender { + cfg := b.cfg.Artifacts + if cfg.BaseURL == "" { + return nil + } + token, err := b.secrets.Resolve(cfg.Token) + if err != nil { + b.log.Warn("artifact publishing disabled: the service credential could not be resolved", + "base_url", cfg.BaseURL, "token_engine", cfg.Token.Engine, "err", err) + return nil + } + if token == "" { + token = b.secrets.Getenv(cfg.TokenEnv) + } + if token == "" { + b.log.Warn("artifact publishing disabled: no service token; set the token secret or its environment variable", + "base_url", cfg.BaseURL, "token_env", cfg.TokenEnv) + return nil + } + return artifactsync.NewClient(cfg.BaseURL, token) +} diff --git a/internal/app/archied/pr_review_test.go b/internal/app/archied/pr_review_test.go index 93384e455..2adc23d84 100644 --- a/internal/app/archied/pr_review_test.go +++ b/internal/app/archied/pr_review_test.go @@ -2,13 +2,16 @@ package archied import ( "context" + "errors" "log/slog" + "strings" "testing" "github.com/samcharles93/archie-core/internal/config" "github.com/samcharles93/archie-core/internal/domain/workflow" "github.com/samcharles93/archie-core/internal/forge" "github.com/samcharles93/archie-core/internal/gateway" + "github.com/samcharles93/archie-core/internal/infrastructure/artifactsync" "github.com/samcharles93/archie-core/internal/tools" "github.com/samcharles93/archie-core/internal/worktree" ) @@ -241,3 +244,74 @@ func hasReviewPRTool(entries []tools.ToolEntry) bool { } return false } + +type fakeArtifactSender struct { + calls int + url string + err error + last artifactsync.Artifact +} + +func (f *fakeArtifactSender) Publish(_ context.Context, a artifactsync.Artifact) (string, error) { + f.calls++ + f.last = a + if f.err != nil { + return "", f.err + } + return f.url, nil +} + +func TestPrReviewerPublishesTheCompletedReview(t *testing.T) { + fake := &fakeArtifactSender{url: "https://offloaded.dev/collab?document=a_1"} + r := &prReviewer{artifacts: fake, log: slog.New(slog.DiscardHandler)} + report := workflow.NewCompletedReviewReport([]workflow.ReviewFinding{{ + File: "handler.go", Line: 42, Defect: "nil deref", + FailureScenario: "empty payload crashes", Verdict: workflow.ReviewVerdictConfirmed, + Level: workflow.ReviewLevelError, + }}, "one finding") + report.Checked = []workflow.ReviewCheck{{Property: "p", Evidence: "e"}} + + url := r.publishArtifact(context.Background(), "system:pr-reviewer", "acme", "widget", 7, report) + if url != fake.url { + t.Fatalf("publishArtifact = %q, want the sender's url %q", url, fake.url) + } + if fake.calls != 1 { + t.Fatalf("Publish called %d times, want 1", fake.calls) + } + got := fake.last + if got.Meta.Format != "markdown" || got.Meta.Category != "pr-review" { + t.Fatalf("format/category = %s/%s, want markdown/pr-review", got.Meta.Format, got.Meta.Category) + } + if got.Meta.Actor != "system:pr-reviewer" || got.Meta.Origin != "pr-review" { + t.Fatalf("attribution = %s/%s", got.Meta.Actor, got.Meta.Origin) + } + if got.Meta.Visibility != "account-only" || got.Meta.Egress != "none" { + t.Fatalf("visibility/egress = %s/%s", got.Meta.Visibility, got.Meta.Egress) + } + if got.Title != "Review of acme/widget#7" { + t.Fatalf("title = %q", got.Title) + } + if got.Meta.RequestID != "artifact:pr-review:acme/widget#7" { + t.Fatalf("requestId = %q, want the deterministic artifact key", got.Meta.RequestID) + } + if got.Meta.Task == nil || got.Meta.Task.Owner != "acme" || got.Meta.Task.Number != 7 { + t.Fatalf("task coordinates = %+v", got.Meta.Task) + } + if !strings.Contains(got.Source, "## Findings") { + t.Fatalf("source is not the rendered report:\n%s", got.Source) + } +} + +func TestPrReviewerPublishDegradesOnSenderFailure(t *testing.T) { + r := &prReviewer{artifacts: &fakeArtifactSender{err: errors.New("503 exhausted")}, log: slog.New(slog.DiscardHandler)} + if url := r.publishArtifact(context.Background(), "system:pr-reviewer", "acme", "widget", 7, workflow.NewCompletedReviewReport(nil, "s")); url != "" { + t.Fatalf("url = %q, want empty on a failed publish", url) + } +} + +func TestPrReviewerPublishIsANoOpWithoutASender(t *testing.T) { + r := &prReviewer{} + if url := r.publishArtifact(context.Background(), "system:pr-reviewer", "acme", "widget", 7, workflow.NewCompletedReviewReport(nil, "s")); url != "" { + t.Fatalf("url = %q, want empty when the sender is disabled", url) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 4eeb1de13..df758e437 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -534,6 +534,10 @@ type Config struct { // Tools holds MCP server and tool policy configuration (from config.tools.yaml). Tools ToolsConfig `toml:"tools" yaml:"tools"` + // Artifacts configures publishing artifacts to the collaborative editor. + // An empty BaseURL disables the sender entirely. + Artifacts ArtifactsConfig `toml:"artifacts" yaml:"artifacts"` + // Identities declares multi-identity configurations. When non-empty, // each identity runs its own poll loop with its own forge client, // worktree manager, repo list, model/provider config, and NATS subject @@ -557,6 +561,20 @@ type Config struct { Extra map[string]any `toml:"-" yaml:"-" json:"extra,omitempty"` } +// ArtifactsConfig locates the collaborative editor that receives artifacts. +// An absent section is valid and disables publishing: the capability degrades +// to a log line rather than stopping the daemon. +type ArtifactsConfig struct { + // BaseURL is the editor origin, e.g. https://offloaded.dev. Empty + // disables artifact publishing. + BaseURL string `toml:"base_url" yaml:"base_url"` + // Token is the service credential the editor's ingest route requires. + Token SecretRef `toml:"token" yaml:"token"` + // TokenEnv names the environment variable holding the token when Token + // is unset. Defaults to WORKSPACE_INGEST_TOKEN. + TokenEnv string `toml:"token_env" yaml:"token_env"` +} + // IdentityConfig is a per-identity configuration subset. Each identity // gets its own poll loop goroutine with isolated forge, repo, model, // and dispatch configuration. diff --git a/internal/domain/workflow/review_report_md.go b/internal/domain/workflow/review_report_md.go new file mode 100644 index 000000000..64cf1d730 --- /dev/null +++ b/internal/domain/workflow/review_report_md.go @@ -0,0 +1,56 @@ +package workflow + +import ( + "fmt" + "strings" +) + +// RenderMarkdown renders an adversarial review report as the artifact source +// the collaborative editor publishes. It is a pure function over the report: +// no I/O, no infrastructure, so the rendered form is testable in isolation. +// +// A completed review renders its summary, every finding as its own subsection +// (file, line, verdict, level, failure scenario, and the mechanical suggestion +// and disposition when present) and the verified properties as a "checked +// clean" list. A review that did not run renders only its reason, because a +// report that never executed has no findings or checks to stand behind. +func (r ReviewReport) RenderMarkdown(title string) string { + var b strings.Builder + fmt.Fprintf(&b, "# %s\n\n", title) + + if r.Status == ReviewStatusNotRun || (r.SkipReason != "" && len(r.Findings) == 0 && len(r.Checked) == 0) { + fmt.Fprintf(&b, "> This review did not run: %s\n", r.SkipReason) + return strings.TrimRight(b.String(), "\n") + "\n" + } + + if r.Summary != "" { + b.WriteString(r.Summary) + b.WriteString("\n\n") + } + + if len(r.Findings) > 0 { + b.WriteString("## Findings\n\n") + for _, f := range r.Findings { + fmt.Fprintf(&b, "### %s:%d — %s (%s)\n\n", f.File, f.Line, f.Level, f.Verdict) + fmt.Fprintf(&b, "**Defect:** %s\n\n", f.Defect) + if f.FailureScenario != "" { + fmt.Fprintf(&b, "**Failure scenario:** %s\n\n", f.FailureScenario) + } + if f.Suggestion != "" { + fmt.Fprintf(&b, "Suggested fix: %s\n\n", f.Suggestion) + } + if f.Disposition != "" { + fmt.Fprintf(&b, "Disposition: %s\n\n", f.Disposition) + } + } + } + + if len(r.Checked) > 0 { + b.WriteString("## Checked clean\n\n") + for _, c := range r.Checked { + fmt.Fprintf(&b, "- %s — %s\n", c.Property, c.Evidence) + } + } + + return strings.TrimRight(b.String(), "\n") + "\n" +} diff --git a/internal/domain/workflow/review_report_md_test.go b/internal/domain/workflow/review_report_md_test.go new file mode 100644 index 000000000..0634930f1 --- /dev/null +++ b/internal/domain/workflow/review_report_md_test.go @@ -0,0 +1,78 @@ +package workflow + +import ( + "strings" + "testing" +) + +func mdFixture() ReviewReport { + disposition := ReviewDisposition("fixed") + return ReviewReport{ + Status: ReviewStatusCompleted, + Summary: "The nil-deref path is reachable from a malformed payload.", + Findings: []ReviewFinding{ + { + File: "handler.go", Line: 42, Defect: "nil deref on empty payload", + FailureScenario: "POST with a body lacking the field crashes the handler.", + Suggestion: "guard the field before use", Verdict: ReviewVerdictConfirmed, + Level: ReviewLevelError, Category: "correctness", Disposition: disposition, + }, + { + File: "store.go", Line: 9, Defect: "query loses the filter", + FailureScenario: "paged reads return rows from other scopes", + Verdict: ReviewVerdictPlausible, Level: ReviewLevelWarn, + }, + }, + Checked: []ReviewCheck{ + {Property: "nil-safety of Foo callers", Evidence: "read all 4 callers; each nil-checks"}, + }, + } +} + +func TestReviewReportRendersFindingsAsMarkdown(t *testing.T) { + got := mdFixture().RenderMarkdown("Review of acme/widget#42") + for _, want := range []string{ + "# Review of acme/widget#42", + "The nil-deref path is reachable", + "### handler.go:42", + "**Defect:** nil deref on empty payload", + "**Failure scenario:** POST with a body lacking the field crashes the handler.", + "error (confirmed)", + "Suggested fix: guard the field before use", + "Disposition: fixed", + "### store.go:9", + "warn (plausible)", + "## Checked clean", + "- nil-safety of Foo callers — read all 4 callers; each nil-checks", + } { + if !strings.Contains(got, want) { + t.Fatalf("rendered report missing %q:\n%s", want, got) + } + } +} + +func TestReviewReportRendersNotRunStatus(t *testing.T) { + report := NewNotRunReviewReport("no reviewer model configured") + got := report.RenderMarkdown("Review of acme/widget#42") + if !strings.Contains(got, "no reviewer model configured") { + t.Fatalf("skip reason missing:\n%s", got) + } + if strings.Contains(got, "## Findings") || strings.Contains(got, "## Checked clean") { + t.Fatalf("a not-run report must not render finding or check sections:\n%s", got) + } +} + +func TestReviewReportRendersACleanReview(t *testing.T) { + report := ReviewReport{ + Status: ReviewStatusCompleted, + Summary: "No defects found.", + Checked: []ReviewCheck{{Property: "p", Evidence: "e"}}, + } + got := report.RenderMarkdown("Review") + if strings.Contains(got, "### ") { + t.Fatalf("a zero-finding review must not render finding sections:\n%s", got) + } + if !strings.Contains(got, "No defects found.") || !strings.Contains(got, "- p — e") { + t.Fatalf("summary and checks missing:\n%s", got) + } +} diff --git a/internal/gateway/review_tools.go b/internal/gateway/review_tools.go index d3e1a83aa..1c42c5dd1 100644 --- a/internal/gateway/review_tools.go +++ b/internal/gateway/review_tools.go @@ -24,6 +24,9 @@ type PRReviewResult struct { // Reason explains why a not_run/skipped review did not execute. Reason string `json:"reason,omitempty"` Findings []PRReviewFinding `json:"findings"` + // ArtifactURL is the editor deep link for the published review artifact, + // empty when artifact publishing is disabled or failed. + ArtifactURL string `json:"artifact_url,omitempty"` } // PRReviewFinding is one defect from an operator-triggered review. diff --git a/internal/infrastructure/artifactsync/sender.go b/internal/infrastructure/artifactsync/sender.go new file mode 100644 index 000000000..c3fee3ae4 --- /dev/null +++ b/internal/infrastructure/artifactsync/sender.go @@ -0,0 +1,181 @@ +// Package artifactsync publishes artifacts to the offloaded.dev collaborative +// editor: archie-core generates and sends, the editor hosts, renders and +// shares. It is infrastructure in the Courier posture — one method, injected +// by the composition root, absent when configuration does not resolve. +package artifactsync + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +const contractFormat = "offloaded-collab-artifact" + +// Artifact is one publishable artifact: the rendered source plus the metadata +// the receiving contract carries. +type Artifact struct { + Title string + Source string // CommonMark for prose, raw source for runnable formats + Meta Meta +} + +// Meta is the artifact block of the receiving contract: shape, attribution and +// idempotency fields. Origin is named "source" on the wire; the top-level +// Source field carries the content. +type Meta struct { + Format string `json:"format"` + Category string `json:"category"` + Entry string `json:"entry,omitempty"` // required for html/react/vue, one file per artifact + Visibility string `json:"visibility"` + Egress string `json:"egress"` + Actor string `json:"actor"` + Origin string `json:"source"` // the producing subsystem, e.g. "pr-review" + RequestID string `json:"requestId"` + Version string `json:"version,omitempty"` + SentAt time.Time `json:"sentAt"` + Task *Task `json:"task,omitempty"` +} + +// Task carries the forge coordinates the artifact was produced for. +type Task struct { + Owner string `json:"owner"` + Repo string `json:"repo"` + Number int `json:"number"` + Task string `json:"task,omitempty"` +} + +// Sender publishes one artifact and returns the editor deep link for it. +type Sender interface { + Publish(ctx context.Context, a Artifact) (string, error) +} + +// Client is the Sender against a deployed editor. +type Client struct { + baseURL string + token string + client *http.Client + // wait is the base delay between 503 retries, doubled per attempt. + wait time.Duration +} + +// NewClient builds a Client against the editor origin. It returns nil when +// baseURL is empty, which is the disabled posture: the composition root skips +// wiring the sender entirely rather than carrying one that cannot be called. +func NewClient(baseURL, token string) *Client { + if baseURL == "" { + return nil + } + return &Client{ + baseURL: strings.TrimRight(baseURL, "/"), + token: token, + client: &http.Client{Timeout: 20 * time.Second}, + wait: 250 * time.Millisecond, + } +} + +// publishBody is the frozen receiving contract. Its shape is the public API; +// changing it is a contract change with the editor, not a refactor. +type publishBody struct { + Format string `json:"format"` + Title string `json:"title"` + Source string `json:"source"` + Artifact Meta `json:"artifact"` +} + +type publishResponse struct { + OK bool `json:"ok"` + ID string `json:"id"` + URL string `json:"url"` +} + +type errorResponse struct { + Error string `json:"error"` +} + +// Publish posts the artifact once. The full source is always sent: versions of +// record are full snapshots stored per publish, never diffs. +func (c *Client) Publish(ctx context.Context, a Artifact) (string, error) { + if c == nil { + return "", fmt.Errorf("artifactsync: sender is not configured") + } + body := publishBody{Format: contractFormat, Title: a.Title, Source: a.Source, Artifact: a.Meta} + payload, err := json.Marshal(body) + if err != nil { + return "", fmt.Errorf("artifactsync: encode artifact: %w", err) + } + + const attempts = 4 // the initial attempt plus three retries + var lastErr error + for attempt := 0; attempt < attempts; attempt++ { + if attempt > 0 { + select { + case <-ctx.Done(): + return "", fmt.Errorf("artifactsync: publish %q: %w", a.RequestID(), ctx.Err()) + case <-time.After(c.wait << (attempt - 1)): + } + } + url, retryable, err := c.postOnce(ctx, payload) + if err == nil { + return url, nil + } + lastErr = err + if !retryable { + return "", err + } + } + return "", fmt.Errorf("artifactsync: publish %q: %w", a.RequestID(), lastErr) +} + +// RequestID is the artifact's own idempotency key, for logging and for callers +// that compose one from the producer's natural key. +func (a Artifact) RequestID() string { return a.Meta.RequestID } + +// postOnce performs one HTTP attempt. retryable marks failures worth another +// attempt: 503 and transport errors. Client errors are configuration or shape +// mistakes and are surfaced immediately with the field reason. +func (c *Client) postOnce(ctx context.Context, payload []byte) (url string, retryable bool, err error) { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+"/api/artifacts", bytes.NewReader(payload)) + if err != nil { + return "", false, fmt.Errorf("artifactsync: build request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+c.token) + + resp, err := c.client.Do(req) + if err != nil { + return "", true, fmt.Errorf("artifactsync: post artifact: %w", err) + } + defer resp.Body.Close() + + switch resp.StatusCode { + case http.StatusCreated: + var out publishResponse + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return "", false, fmt.Errorf("artifactsync: decode publish response: %w", err) + } + if out.URL == "" { + return "", false, fmt.Errorf("artifactsync: publish response carries no url") + } + return out.URL, false, nil + case http.StatusServiceUnavailable: + return "", true, fmt.Errorf("artifactsync: editor store unavailable (503)") + case http.StatusUnauthorized: + return "", false, fmt.Errorf("artifactsync: editor rejected the service token (401); check [artifacts].token") + case http.StatusBadRequest: + reason := "unknown" + raw, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + var e errorResponse + if json.Unmarshal(raw, &e) == nil && e.Error != "" { + reason = e.Error + } + return "", false, fmt.Errorf("artifactsync: editor rejected the artifact (400): %s", reason) + default: + return "", true, fmt.Errorf("artifactsync: unexpected publish status %d", resp.StatusCode) + } +} diff --git a/internal/infrastructure/artifactsync/sender_test.go b/internal/infrastructure/artifactsync/sender_test.go new file mode 100644 index 000000000..930cf725c --- /dev/null +++ b/internal/infrastructure/artifactsync/sender_test.go @@ -0,0 +1,159 @@ +package artifactsync + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" +) + +func sampleArtifact() Artifact { + return Artifact{ + Title: "Review of PR #7", + Source: "# Review\n\none finding, one check", + Meta: Meta{ + Format: "markdown", + Category: "pr-review", + Visibility: "account-only", + Egress: "none", + Actor: "system:pr-reviewer", + Origin: "pr-review", + RequestID: "artifact:pr-review:acme/widget#7", + Version: "r1", + SentAt: time.Date(2026, 9, 22, 16, 45, 0, 0, time.UTC), + Task: &Task{Owner: "acme", Repo: "widget", Number: 7}, + }, + } +} + +func TestPublishPostsTheContractBody(t *testing.T) { + var gotPath, gotAuth string + var gotBody map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath, gotAuth = r.URL.Path, r.Header.Get("Authorization") + body, _ := io.ReadAll(r.Body) + _ = json.Unmarshal(body, &gotBody) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte(`{"ok":true,"id":"a_1","url":"https://offloaded.dev/collab?document=a_1"}`)) + })) + defer srv.Close() + + c := NewClient(srv.URL, "secret-token") + url, err := c.Publish(context.Background(), sampleArtifact()) + if err != nil { + t.Fatalf("Publish: %v", err) + } + if url != "https://offloaded.dev/collab?document=a_1" { + t.Fatalf("url = %q, want the editor deep link", url) + } + if gotPath != "/api/artifacts" { + t.Fatalf("path = %q, want /api/artifacts", gotPath) + } + if gotAuth != "Bearer secret-token" { + t.Fatalf("Authorization = %q, want the bearer token", gotAuth) + } + if gotBody["format"] != "offloaded-collab-artifact" { + t.Fatalf("top-level format = %v, want the contract envelope", gotBody["format"]) + } + if gotBody["title"] != "Review of PR #7" || gotBody["source"] != sampleArtifact().Source { + t.Fatalf("title/source did not round-trip: %v", gotBody) + } + artifact, ok := gotBody["artifact"].(map[string]any) + if !ok { + t.Fatalf("artifact block missing: %v", gotBody) + } + if artifact["format"] != "markdown" || artifact["category"] != "pr-review" { + t.Fatalf("artifact format/category = %v", artifact) + } + if artifact["actor"] != "system:pr-reviewer" || artifact["source"] != "pr-review" { + t.Fatalf("attribution = %v", artifact) + } + if artifact["requestId"] != "artifact:pr-review:acme/widget#7" || artifact["version"] != "r1" { + t.Fatalf("idempotency fields = %v", artifact) + } + if artifact["visibility"] != "account-only" || artifact["egress"] != "none" { + t.Fatalf("visibility/egress = %v", artifact) + } + task, ok := artifact["task"].(map[string]any) + if !ok || task["owner"] != "acme" || task["repo"] != "widget" { + t.Fatalf("task coordinates = %v", artifact["task"]) + } +} + +func TestPublishDoesNotRetryAClientError(t *testing.T) { + for _, status := range []int{http.StatusUnauthorized, http.StatusBadRequest} { + var attempts atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + attempts.Add(1) + w.WriteHeader(status) + _, _ = w.Write([]byte(`{"error":"source is required"}`)) + })) + _, err := NewClient(srv.URL, "tok").Publish(context.Background(), sampleArtifact()) + srv.Close() + if err == nil { + t.Fatalf("status %d: Publish error = nil, want failure", status) + } + if attempts.Load() != 1 { + t.Fatalf("status %d attempts = %d, want exactly one attempt", status, attempts.Load()) + } + } +} + +func TestPublishSurfacesTheFieldReasonOn400(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":"source is required"}`)) + })) + defer srv.Close() + _, err := NewClient(srv.URL, "tok").Publish(context.Background(), sampleArtifact()) + if err == nil || !strings.Contains(err.Error(), "source is required") { + t.Fatalf("error = %v, want it to carry the field reason", err) + } +} + +func TestPublishRetries503UntilSuccess(t *testing.T) { + var attempts atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if attempts.Add(1) < 3 { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte(`{"ok":true,"id":"a_2","url":"u"}`)) + })) + defer srv.Close() + + c := NewClient(srv.URL, "tok") + c.wait = time.Millisecond + url, err := c.Publish(context.Background(), sampleArtifact()) + if err != nil || url != "u" { + t.Fatalf("Publish = (%q, %v), want success after two 503s", url, err) + } + if attempts.Load() != 3 { + t.Fatalf("attempts = %d, want 3", attempts.Load()) + } +} + +func TestPublishFailsWhen503Persists(t *testing.T) { + var attempts atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + attempts.Add(1) + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer srv.Close() + + c := NewClient(srv.URL, "tok") + c.wait = time.Millisecond + if _, err := c.Publish(context.Background(), sampleArtifact()); err == nil { + t.Fatal("Publish error = nil, want failure after exhausting retries") + } + if attempts.Load() != 4 { + t.Fatalf("attempts = %d, want the initial attempt plus three retries", attempts.Load()) + } +} diff --git a/internal/infrastructure/configuration/artifacts_config_test.go b/internal/infrastructure/configuration/artifacts_config_test.go new file mode 100644 index 000000000..a7bef8923 --- /dev/null +++ b/internal/infrastructure/configuration/artifacts_config_test.go @@ -0,0 +1,52 @@ +package configuration + +import ( + "os" + "path/filepath" + "testing" + + "github.com/samcharles93/archie-core/internal/config" +) + +func TestArtifactsConfigDefaults(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.toml") + contents := "bot_user = \"widget\"\n" + + "[forge]\ntype = \"gitea\"\nhost = \"https://git.example.test\"\ntoken_env = \"T\"\n" + if err := os.WriteFile(path, []byte(contents), 0o600); err != nil { + t.Fatal(err) + } + cfg, err := loadFile(path) + if err != nil { + t.Fatal(err) + } + if cfg.Artifacts.BaseURL != "" { + t.Errorf("BaseURL = %q, want empty so the sender stays disabled", cfg.Artifacts.BaseURL) + } + if cfg.Artifacts.TokenEnv != "WORKSPACE_INGEST_TOKEN" { + t.Errorf("TokenEnv = %q, want the documented fallback", cfg.Artifacts.TokenEnv) + } +} + +func TestArtifactsConfigPreservesExplicitValues(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.toml") + contents := "bot_user = \"widget\"\n" + + "[artifacts]\nbase_url = \"https://offloaded.dev\"\n" + + "token = { engine = \"env\", key = \"ARCHIE_GITHUB_TOKEN\" }\n" + + "token_env = \"OTHER_TOKEN\"\n" + if err := os.WriteFile(path, []byte(contents), 0o600); err != nil { + t.Fatal(err) + } + cfg, err := loadFile(path) + if err != nil { + t.Fatal(err) + } + if cfg.Artifacts.BaseURL != "https://offloaded.dev" { + t.Errorf("BaseURL = %q, want the explicit value", cfg.Artifacts.BaseURL) + } + if want := (config.SecretRef{Engine: "env", Key: "ARCHIE_GITHUB_TOKEN"}); cfg.Artifacts.Token != want { + t.Errorf("Token = %+v, want %+v", cfg.Artifacts.Token, want) + } + if cfg.Artifacts.TokenEnv != "OTHER_TOKEN" { + t.Errorf("TokenEnv = %q, want the explicit value", cfg.Artifacts.TokenEnv) + } +} diff --git a/internal/infrastructure/configuration/defaults.go b/internal/infrastructure/configuration/defaults.go index a328625ba..18344a82f 100644 --- a/internal/infrastructure/configuration/defaults.go +++ b/internal/infrastructure/configuration/defaults.go @@ -76,6 +76,7 @@ func (l *Loader) applyDefaults(cfg *config.Config) { } l.applyGeneralDefaults(cfg) applyForgeDefaults(cfg) + applyArtifactsDefaults(cfg) applyDispatchDefaults(cfg) applyIdentityDefaults(cfg) applyContainerDefaults(cfg) @@ -279,6 +280,17 @@ func applyForgeDefaults(cfg *config.Config) { } } +// applyArtifactsDefaults fills the artifacts section, promoting the +// token_env fallback to a secret reference when no explicit one was given. +func applyArtifactsDefaults(cfg *config.Config) { + if cfg.Artifacts.TokenEnv == "" { + cfg.Artifacts.TokenEnv = "WORKSPACE_INGEST_TOKEN" + } + if cfg.Artifacts.Token == (config.SecretRef{}) { + cfg.Artifacts.Token = config.SecretRef{Engine: "env", Key: cfg.Artifacts.TokenEnv} + } +} + // applyNATSDefaults resolves the nats mode from URL when the operator left it // unset. URL set implies external; URL empty implies embedded. func applyNATSDefaults(cfg *config.Config) { From 236974fbae68ff0450b34ff16dd6077dc4792641 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Wed, 23 Sep 2026 03:16:35 +1000 Subject: [PATCH 17/20] chore(docs): regenerate dev-docs.json for the artifact-sender PRD e0becd52 added docs/prds/artifact-sender.md without regenerating the artifact, so docs:artifact:check failed at that commit. Generated from a clean worktree at e0becd52. --- docs/data/generated/dev-docs.json | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/data/generated/dev-docs.json b/docs/data/generated/dev-docs.json index 2c85a1f3f..f8286d4c4 100644 --- a/docs/data/generated/dev-docs.json +++ b/docs/data/generated/dev-docs.json @@ -126,6 +126,12 @@ "section": "PRDs", "body": "# Agent2Agent as archie's agent-to-agent protocol — decision\n\n**Status:** Draft\n**Date:** 2026-09-22\n**Authority:** the A2A Protocol Specification v1.0.0 (`a2a-protocol.org`) for the\nprotocol, `docs/architecture/organisation.md` for package placement, and\n`docs/architecture/migration-decisions.md` §2 for session and messaging design.\n\n## Decision\n\nA2A is the protocol archie speaks to other agents. It carries agent-to-agent\nwork: discovery of what a remote agent offers, delegation of a unit of work to it,\nprogress on that work, and the artifacts it produces.\n\n## Layers\n\n**MCP stays the tool layer.** MCP connects a model to tools and data; A2A connects\nagents to agents. They are complementary and archie runs both:\n`internal/tools/provider` registers MCP providers as optional capabilities, so an\nA2A call and an MCP tool call are different kinds of traffic with different\nowners.\n\n**A2A rides archie's existing transports.** A2A's canonical data model is\nProtocol Buffer messages and it defines a gRPC binding, so archie speaks A2A over\nthe gRPC and NATS it already runs. A second wire format, a second broker, or a\nbespoke protocol for agent-to-agent traffic is a requirement not to introduce.\n\n## Where A2A stops\n\nA2A is not a general replacement for either of the two protocols archie already\nhas, and each boundary is a requirement rather than a preference.\n\n- **`internal/domain/messaging` stays the human and channel layer.** A\n Conversation is a ChannelID and a ThreadID, its participants are people on\n Telegram, email, webhook and the dashboard, and its records are what a curator\n reads. An A2A peer is not a channel participant and does not address a\n Conversation.\n- **`internal/agentexec` stays the daemon-to-worker execution protocol.** It\n carries a Request, a Result, gate and budget policy, and capture tools between\n the daemon and the agent container it owns. An A2A peer never receives that\n protocol, and A2A never carries gate, budget or protection policy.\n\nA2A is therefore an edge protocol: it exchanges work with peers outside archie's\nown process boundary, and it composes with the two protocols above rather than\nreplacing or abstracting them.\n\n## A2A Task mapping\n\n**Decision: archie's work lifecycle is mapped to A2A's, not expressed in it. An\nA2A Task is a view over archie's own task, and archie's status remains the record\nof truth.**\n\nThis is the load-bearing decision, because A2A's Task carries its own state\nmachine and archie's lifecycle is the on-disk format of its task store, whose\nstatus strings a rename migrates rather than changes.\n\nThe mapping is partial, and each gap is a case where archie knows something A2A's\nstates do not carry:\n\n- **Completion by another system.** A2A's terminal states describe what the agent\n did (`completed`, `failed`, `canceled`, `rejected`). archie's `merged` and\n `rejected` describe what a forge did to a pull request after archie finished\n its work. There is no A2A state for \"an external system accepted this\", so a\n completed A2A task cannot express the outcome that matters most to the caller.\n- **Handing work to a reviewer.** archie's `pr_open` means the work left\n archie's hands for human review. A2A's `input-required` is the nearest state,\n but it means the *client* must send more input, and a reviewer is not the\n client.\n- **Operator decisions.** `declined`, `parked` and `dead` are outcomes of\n archie's own policy and operator actions. A2A has no state for \"the operator\n refused this\" or \"a budget ran out and an operator must decide\".\n\nA WorkflowExecution is the unit an A2A task maps onto, because it is the attempt\nthat runs, streams and produces artifacts, and a follow-up attempt is a new\nexecution. A2A's own model agrees that a task is immutable once terminal and that\na refinement is a new task sharing the `contextId`, which is why archie's\nper-attempt executions and A2A's per-request tasks correspond.\n\n**Requirement:** the A2A state a peer sees is derived from archie's status by one\nmapping, and archie's status is carried on the task so a peer can read what the\nderivation lost. The mapping is total in the direction that matters -- every\narchie status has an A2A state -- and deliberately not injective, because the\ngaps above are facts A2A cannot hold.\n\n**Requirement:** adopting A2A's state names as archie's persisted task status is\nrejected. It would migrate the task store, and it would drop the forge outcomes\nthat tell a caller whether the work was accepted.\n\n## What an A2A agent is\n\n**An A2A agent is an archie Identity.** The Agent Card describes a durable,\naddressable actor with an endpoint, capabilities and authentication requirements,\nand an Identity is the actor archie persists, versions, suspends and scopes\ncredentials to.\n\nAn `agent.Persona` is not an A2A agent: a persona is prompt text selected for a\nturn, and it has no endpoint or credential. An agent container is not an A2A\nagent: it is task-scoped and ephemeral, and it terminates with the work it ran.\nThe daemon is not an A2A agent: it is the process that serves the identity.\n\n## The Agent Card\n\n**The Gateway serves the card, for the identity it is configured as.** A2A's\n\"Get Agent Card\" is an abstract operation with a gRPC binding, so the card is\nserved through the binding archie runs and needs no HTTP listener.\n\n**The card is derived, never hand-written.** Its skills come from the skill\ncatalog, its identity from the configured Identity, and its capabilities from\nwhat the process already advertises about itself. A card that restated\ncapabilities independently would be a second inventory of the same facts, which\nis the class of drift the existing capability reporting exists to prevent.\n\n**The card advertises the authentication archie applies**, and it carries no\nsecret. A card that named a token would be a credential published at a\nwell-known location.\n\n## Server before client\n\n**archie is an A2A server first.** Accepting A2A calls is the smaller direction\nand the more valuable one: the work a peer would delegate -- an issue taken to a\npull request, through archie's gates, worktrees and review -- exists only inside\narchie, and archie already serves its own contracts over gRPC to the Messaging\nService, the dashboard and the State Store. Acting as an A2A client to remote\nagents is a later capability, and one that consumes the same contract.\n\n## Authentication and authorization\n\n**An inbound A2A call is authenticated at the transport, by the same bearer-token\ninterceptor the Gateway applies to its other contracts.** A2A carries no identity\nin its payload by design: identity is established where the connection is, so the\ntransport token is the identity and no field in a Message or Task names a caller.\n\n**Authorization is per skill**, which is A2A's own model: the card advertises\nskills, and a caller may invoke the skills its credential is scoped to.\n\n**Task-scoped grants are not A2A credentials.** They scope a worker's access to\none task's records inside the State Store and are issued by the daemon for its own\ncontainers. Reusing one as an inbound A2A credential would let a peer act with a\nworker's authority, which is the isolation the grants exist to enforce.\n\n**The dashboard's authentication is not an A2A scheme.** It gates an operator\ninterface, not an endpoint that agents call, so it authenticates no A2A request.\n\n## Package placement\n\n- The contract -- A2A's data model, operations and the mapping from archie's\n status -- lives in `internal/domain/\u003carea\u003e/`.\n- The binding that carries it over gRPC lives in `internal/infrastructure/`,\n beside the existing gRPC clients and servers.\n- Wiring of the contract to the Gateway and the task store lives in\n `internal/app/`.\n\n## Verification\n\n- A peer fetches the Agent Card over gRPC and receives the configured identity's\n identity, skills and authentication requirements.\n- A delegated task reaches an archie task, and the A2A state a peer observes\n changes as archie's status changes, through the one mapping.\n- A task that reaches `merged` reports an A2A state derived from it, and the\n archie status is readable on the task.\n- A call with no credential, and a call with a credential scoped to another\n skill, are both refused, with the refusal A2A defines for them.\n- A task-scoped grant presented as an inbound A2A credential is refused.\n- No A2A traffic reaches `internal/domain/messaging` or `internal/agentexec`.\n" }, + { + "url": "/dev/prds/artifact-sender/", + "title": "Artifact sender", + "section": "PRDs", + "body": "# Artifact sender\n\n**Status:** Approved\n**Revision:** 1\n**Authority:** this document\n\n## Purpose\n\narchie-core publishes artifacts — design documents, review reports, research briefs,\nrunnable pages — to the offloaded.dev collaborative editor. A person reads the\nartifact, comments on it, and the artifact page is the shared surface for iteration.\n\nThe receiving contract is owned by the editor: `POST /api/artifacts` with a bearer\nservice token, a closed field set, and server-side idempotency on `requestId` plus\nlast-write-wins on `sentAt`. This document owns only what archie-core sends.\n\n## Scope\n\nVersion one publishes one-way: archie-core sends, the editor receives. The first\nproducer is the pull-request adversarial review: a completed `ReviewReport` is\nrendered to markdown and published after the review completes.\n\nChat-surface tools (`create_artifact`, `update_artifact`), comment read-back, share\ntokens, and runnable-format producers (`html`, `react`, `vue`) are later slices. The\nreceiving contract already carries them; version one does not exercise them.\n\n## Configuration\n\n- New `[artifacts]` section: `base_url` (site origin), `token` (`SecretRef`) with\n `token_env` fallback naming `WORKSPACE_INGEST_TOKEN`.\n- An empty `base_url` disables the sender: no client is constructed, nothing is\n registered, and publishing degrades to a log line.\n- The token is runtime configuration. It is never baked at build time.\n\n## Sender\n\n- New package `internal/infrastructure/artifactsync` exposing one method,\n `Publish(ctx, Artifact) (string, error)`, where the string is the artifact URL.\n One-method injected seam, concrete client constructed by the boot path when the\n configuration resolves — the `crondelivery.Courier` posture.\n- The client POSTs the frozen contract body. Each publish sends the complete\n source: versions of record are full snapshots, never diffs.\n- A `503` response is retried three times with backoff. `401` and `400` are not\n retried; they are logged with the field reason the response names.\n- `requestId` is `artifact:\u003ctask-id\u003e:\u003cversion\u003e` when the producer is task-backed,\n otherwise `artifact:\u003cidentity\u003e:\u003csentAt-utc\u003e`. Sending the same `requestId` with\n an equal or older `sentAt` must be a server-side no-op, which makes sender-side\n retry safe.\n\n## Report rendering\n\n- `ReviewReport` gains a pure markdown renderer in `internal/domain/workflow`:\n the summary as the lede, one finding per subsection with file, line, verdict,\n level and the failure scenario, and the verified properties as a \"checked\n clean\" list. Findings carry their disposition when present.\n- The renderer performs no I/O and imports no infrastructure.\n\n## Attribution\n\n- `actor`: the producing identity. The reviewer publishes as `system:pr-reviewer`\n when no task identity resolves; otherwise the task's identity name.\n- `source`: the producing subsystem, `pr-review` for version one.\n- `task`: owner, repository and issue number from the task when one exists.\n\n## Failure behaviour\n\n- Publishing never fails the producing work. A failed publish is logged with its\n status and reason; the artifact is republished on the next production of the\n same artifact.\n- A successful publish records the artifact URL in the run report, so the PR\n description or task log links to the readable artifact.\n- The published URL is the editor deep link for the artifact, keyed on its\n immutable id. archie-core never mints share links; sharing is an\n account-holder decision made in the editor.\n\n## Verification\n\n- Client tests against a test server cover 201, 401, 400 with a field reason,\n and 503 retry behaviour.\n- The report renderer test covers a report with findings, checks, dispositions\n and a not-run status.\n- Configuration tests cover the disabled case, the `SecretRef` path and the\n `token_env` fallback." + }, { "url": "/dev/prds/binding-secret-encryption/", "title": "Binding secret at-rest encryption -- decision", @@ -517,6 +523,10 @@ "title": "Agent2Agent as archie's agent-to-agent protocol — decision", "url": "/dev/prds/agent2agent-protocol/" }, + { + "title": "Artifact sender", + "url": "/dev/prds/artifact-sender/" + }, { "title": "Binding secret at-rest encryption -- decision", "url": "/dev/prds/binding-secret-encryption/" From 019c681137214a4887a4e7ab00e243f5a1a80fc0 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Wed, 23 Sep 2026 03:23:51 +1000 Subject: [PATCH 18/20] fix(daemon): fail closed on unresolvable task identity --- internal/daemon/daemon.go | 46 ++++++ internal/daemon/identity_dispatch_test.go | 163 ++++++++++++++++++++++ 2 files changed, 209 insertions(+) create mode 100644 internal/daemon/identity_dispatch_test.go diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index 7add5e30e..bef1d5e2f 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -994,6 +994,14 @@ func (d *Daemon) processNATSTask(ctx context.Context, msg eventbus.Message) { return } + if ok, reason := d.identityMayAct(ctx, tm.Identity); !ok { + d.Log.Error("nats intake rejected: "+reason, "subject", msg.Subject()) + if err := msg.Ack(); err != nil { + d.Log.Warn("ack failed", "err", err) + } + return + } + inserted, err := d.Store.EnqueueIssue(ctx, tm.Owner, tm.Repo, tm.Number, tm.Title, tm.Body, strings.Join(tm.Labels, ","), tm.Identity) if err != nil { d.Log.Error("nats enqueue failed", "err", err) @@ -1126,6 +1134,10 @@ func (d *Daemon) process(ctx context.Context, task *workflow.Task) { defer finished() defer d.openTaskLog(task)() + if d.parkIfIdentityUnresolvable(ctx, task) { + return + } + trees := d.treesFor(task) repo, ok := d.repoFor(task) if !ok { @@ -1770,6 +1782,40 @@ func (d *Daemon) identityFor(task *workflow.Task) *IdentityRunner { return nil } +// identityMayAct reports whether the identity a task carries may be acted on. +// An empty identity is the single-identity root and always may act. A +// non-empty identity must resolve to a configured runner and pass the +// lifecycle check; otherwise the returned reason says why it may not. This is +// the single task-side fail-closed point, mirroring identity.Resolve's +// credential-side behaviour: a task naming an identity archie no longer knows +// -- renamed, retired, or deleted in the control plane -- must park, never +// fall back to the root forge's credential. +func (d *Daemon) identityMayAct(ctx context.Context, name string) (bool, string) { + if name == "" { + return true, "" + } + runner := d.identityFor(&workflow.Task{Identity: name}) + if runner == nil { + return false, "identity " + name + " no longer resolves" + } + if !d.identityActive(ctx, runner.ID) { + return false, "identity " + runner.Name + " may not act" + } + return true, "" +} + +// parkIfIdentityUnresolvable parks task when its non-empty identity cannot be +// acted on, reporting whether it did. An empty identity is the root +// single-identity path and never parks here. +func (d *Daemon) parkIfIdentityUnresolvable(ctx context.Context, task *workflow.Task) bool { + ok, reason := d.identityMayAct(ctx, task.Identity) + if ok { + return false + } + d.parkRunningTask(ctx, task.ID, reason) + return true +} + func (d *Daemon) identityActive(ctx context.Context, id identity.IdentityID) bool { if d.IdentityRepository == nil { return true diff --git a/internal/daemon/identity_dispatch_test.go b/internal/daemon/identity_dispatch_test.go new file mode 100644 index 000000000..0bcc6b839 --- /dev/null +++ b/internal/daemon/identity_dispatch_test.go @@ -0,0 +1,163 @@ +package daemon + +import ( + "context" + "log/slog" + "strings" + "testing" + + "github.com/samcharles93/archie-core/internal/config" + "github.com/samcharles93/archie-core/internal/domain/identity" + "github.com/samcharles93/archie-core/internal/domain/workflow" + "github.com/samcharles93/archie-core/internal/domain/workintake" + "github.com/samcharles93/archie-core/internal/eventbus" + "github.com/samcharles93/archie-core/internal/store" + "github.com/samcharles93/archie-core/internal/worktree" +) + +// fakeMessage is a minimal eventbus.Message for driving processNATSTask +// without standing up a broker. +type fakeMessage struct { + data []byte + subject string + acked bool + naked bool +} + +func (m *fakeMessage) Data() []byte { return m.data } +func (m *fakeMessage) Subject() string { return m.subject } +func (m *fakeMessage) Ack() error { m.acked = true; return nil } +func (m *fakeMessage) Nak() error { m.naked = true; return nil } + +var _ eventbus.Message = (*fakeMessage)(nil) + +// TestProcessNATSTaskRejectsUnresolvableIdentity pins the intake half of the +// fail-closed relation: a task envelope naming an identity this daemon does +// not know must not be written to the store at all. Today it is enqueued and +// only fails open later, when forgeFor/repoFor fall back to the root forge. +func TestProcessNATSTaskRejectsUnresolvableIdentity(t *testing.T) { + s := store.OpenTest(t) + d := &Daemon{ + Store: s, + Cfg: config.NewHolder(config.Config{}), + Trees: &worktree.Manager{}, + Log: slog.New(slog.DiscardHandler), + } + + data, err := workintake.TaskEnvelope{ + Owner: "acme", Repo: "widget", Number: 1, Title: "ghost task", + Identity: "ghost", + }.Encode() + if err != nil { + t.Fatal(err) + } + msg := &fakeMessage{data: data, subject: workintake.SubjectTaskDefault} + + d.processNATSTask(t.Context(), msg) + + if msg.naked { + t.Fatal("unresolvable identity intake was nacked; a redelivery can never resolve it") + } + if !msg.acked { + t.Fatal("unresolvable identity intake was not dropped (acked)") + } + task, err := s.TaskByIssue(t.Context(), "acme", "widget", 1) + if err != nil { + t.Fatal(err) + } + if task != nil { + t.Fatalf("task %d was created with an unresolvable identity %q", task.ID, task.Identity) + } +} + +// TestProcessParksTaskWhoseIdentityNoLongerResolves pins the dispatch half of +// the fail-closed relation: a task whose non-empty identity no longer has a +// configured runner must park, never fall back to the root forge's credential. +func TestProcessParksTaskWhoseIdentityNoLongerResolves(t *testing.T) { + d, s, rootFg, _, _ := twoIdentityDaemon(t) + d.Log = slog.New(slog.DiscardHandler) + ctx := context.Background() + + if _, err := s.EnqueueIssue(ctx, "acme", "shared", 42, "t", "b", "", "ghost"); err != nil { + t.Fatal(err) + } + task, err := s.ClaimNext(ctx) + if err != nil || task == nil { + t.Fatalf("ClaimNext = (%+v, %v)", task, err) + } + + d.process(ctx, task) + + got, err := s.TaskByID(ctx, task.ID) + if err != nil || got == nil { + t.Fatalf("TaskByID = (%+v, %v)", got, err) + } + if got.Status != workflow.StatusParked { + t.Fatalf("status = %q, want parked", got.Status) + } + if !strings.Contains(got.ParkReason, "no longer resolves") { + t.Errorf("ParkReason = %q, want it to name the unresolvable identity", got.ParkReason) + } + // The root forge must not have been touched for this task: parking on an + // identity that no longer resolves is the fail-closed outcome, not a + // fallback to root credentials. + if calls := len(rootFg.stateLabels) + len(rootFg.closedIssues) + len(rootFg.comments); calls != 0 { + t.Errorf("root forge was called %d time(s) for a task with an unresolvable identity", calls) + } +} + +// TestProcessParksRetiredIdentity pins the lifecycle half: an identity that +// still has a runner but has been retired may not act, so its tasks park +// rather than running under a credential the control plane has retired. +func TestProcessParksRetiredIdentity(t *testing.T) { + s := store.OpenTest(t) + ctx := context.Background() + log := slog.New(slog.DiscardHandler) + + id := identity.StableID("worker") + created, err := identity.New(id, identity.KindBot, "worker") + if err != nil { + t.Fatal(err) + } + created, err = s.Create(ctx, created, identity.Audit{ActorID: identity.SystemID, Source: "test", RequestID: "create"}) + if err != nil { + t.Fatal(err) + } + if _, err := s.Apply(ctx, created.ID, created.Version, identity.Command{Type: identity.CommandRetire}, identity.Audit{ActorID: identity.SystemID, Source: "test", RequestID: "retire"}); err != nil { + t.Fatal(err) + } + + repo := config.Repo{Owner: "acme", Name: "widget"} + d := &Daemon{ + Cfg: config.NewHolder(config.Config{Repos: []config.Repo{repo}}), + Store: s, + Log: log, + IdentityRepository: s, + Identities: []*IdentityRunner{{ + Name: "worker", ID: id, + Forge: &testForge{}, Trees: &worktree.Manager{}, + Repos: []config.Repo{repo}, Cfg: config.IdentityConfig{Name: "worker"}, Log: log, + }}, + } + + if _, err := s.EnqueueIssue(ctx, "acme", "widget", 7, "t", "b", "", string(id)); err != nil { + t.Fatal(err) + } + task, err := s.ClaimNext(ctx) + if err != nil || task == nil { + t.Fatalf("ClaimNext = (%+v, %v)", task, err) + } + + d.process(ctx, task) + + got, err := s.TaskByID(ctx, task.ID) + if err != nil || got == nil { + t.Fatalf("TaskByID = (%+v, %v)", got, err) + } + if got.Status != workflow.StatusParked { + t.Fatalf("status = %q, want parked", got.Status) + } + if !strings.Contains(got.ParkReason, "may not act") { + t.Errorf("ParkReason = %q, want it to name the retired identity", got.ParkReason) + } +} From a0d6cfbb286a8b97eeffc736bcb3cf877e0ae66d Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Wed, 23 Sep 2026 03:26:14 +1000 Subject: [PATCH 19/20] feat(edastore): project tool_call events into the tool_calls collection The tool_calls collection existed in the event-capture schema but nothing wrote it. Give it its write path: - edastore gains InsertToolCall and the minimal TaskToolCalls read, writing only the existing collection. No new index: the collection is an append-only transcript and the SQLite predecessor had no unique index. - internal/app/archied composes toolCallProjectingTaskStore, a decorator on the task store that persists the event first, then -- only for tool_call events -- writes one row. A projection failure is counted and logged but never returned from InsertEvent: event persistence is load-bearing, tool_calls is observability. - stateStoreDeps wires the decorator around deps.Tasks when both stores are open, and passes Tasks through untouched otherwise. The event adapts to the existing schema, not the other way round: the tool name comes from data, a succeeded call's detail is the row's result, a failed call's detail its error; args and duration_ms stay reserved for a producer that reports them. --- internal/app/archied/state_store.go | 9 + internal/app/archied/state_store_toolcalls.go | 87 ++++++++ .../app/archied/state_store_toolcalls_test.go | 192 ++++++++++++++++++ .../infrastructure/edastore/tool_calls.go | 98 +++++++++ .../edastore/tool_calls_test.go | 102 ++++++++++ 5 files changed, 488 insertions(+) create mode 100644 internal/app/archied/state_store_toolcalls.go create mode 100644 internal/app/archied/state_store_toolcalls_test.go create mode 100644 internal/infrastructure/edastore/tool_calls.go create mode 100644 internal/infrastructure/edastore/tool_calls_test.go diff --git a/internal/app/archied/state_store.go b/internal/app/archied/state_store.go index 1756b2506..4918ee7b5 100644 --- a/internal/app/archied/state_store.go +++ b/internal/app/archied/state_store.go @@ -300,6 +300,15 @@ func (b *boot) stateStoreDeps(grants *staterpc.TaskGrants) staterpc.Deps { deps.BindingDispatcher = b.eda deps.PlaybookDispatcher = b.eda } + // tool_call events project into the tool_calls collection on the same + // event-capture store: this process legitimately owns both, so the + // projection rides on the task-store surface it decorates. With no + // event-capture store (or no task store) there is nothing to project + // through, and Tasks passes through unwrapped rather than decorating a + // nil operand. + if b.eda != nil && b.st != nil { + deps.Tasks = newToolCallProjectingTaskStore(b.st, b.eda, b.log) + } if btc, ok := b.st.(storecontract.BindingTaskCreator); ok { deps.BindingTaskCreator = btc } diff --git a/internal/app/archied/state_store_toolcalls.go b/internal/app/archied/state_store_toolcalls.go new file mode 100644 index 000000000..f4ba41edc --- /dev/null +++ b/internal/app/archied/state_store_toolcalls.go @@ -0,0 +1,87 @@ +package archied + +import ( + "context" + "log/slog" + "sync/atomic" + + "github.com/samcharles93/archie-core/internal/domain/storecontract" + "github.com/samcharles93/archie-core/internal/events" + "github.com/samcharles93/archie-core/internal/infrastructure/edastore" +) + +// toolCallWriter is the edastore surface the projection writes through. Kept +// narrow so a test can fail it without touching the real store. +type toolCallWriter interface { + InsertToolCall(ctx context.Context, tc edastore.ToolCall) error +} + +// toolCallProjectingTaskStore decorates the State Store's task store so every +// persisted tool_call event also lands in the tool_calls collection on the +// event-capture store, which this same process owns. The task event log stays +// the source of truth; the collection is the queryable transcript of it. +// +// A projection failure is NEVER returned: event persistence is load-bearing, +// the collection is observability, and failing a task write because its +// transcript copy failed would trade the fact for its shadow. The failure is +// instead counted and logged with the running count, so it is visible without +// ever changing InsertEvent's outcome. +type toolCallProjectingTaskStore struct { + storecontract.TaskStore + toolCalls toolCallWriter + log *slog.Logger + projectionFailures atomic.Int64 +} + +func newToolCallProjectingTaskStore(inner storecontract.TaskStore, toolCalls toolCallWriter, log *slog.Logger) *toolCallProjectingTaskStore { + if log == nil { + log = slog.Default() + } + return &toolCallProjectingTaskStore{TaskStore: inner, toolCalls: toolCalls, log: log} +} + +// toolCallProjectionFailures reports how many tool_call projections have been +// dropped since this process started. Named and exported to the test surface +// rather than buried in a log field alone: a silent swallow is the bug this +// decorator exists not to have. +func (s *toolCallProjectingTaskStore) toolCallProjectionFailures() int64 { + return s.projectionFailures.Load() +} + +func (s *toolCallProjectingTaskStore) InsertEvent(ctx context.Context, e events.Event) (int64, error) { + id, err := s.TaskStore.InsertEvent(ctx, e) + if err != nil || e.Kind != events.KindToolCall { + return id, err + } + if perr := s.toolCalls.InsertToolCall(ctx, toolCallFromEvent(e)); perr != nil { + failures := s.projectionFailures.Add(1) + s.log.Error("tool_call projection failed; the event persisted but the tool_calls row did not", + "err", perr, "task_id", e.TaskID, "attempt", e.Attempt, + "tool", e.Data["tool"], "projection_failures", failures) + } + return id, nil +} + +// toolCallFromEvent adapts a tool_call event to the existing tool_calls +// collection. The event carries the tool name and the failure flag in data and +// the one-line outcome in detail: a succeeded call's summary is its result, a +// failed call's summary is its error. The collection's args and duration_ms +// fields stay unwritten -- the event reports neither -- so they are schema +// reserved for a producer that one day reports them, not promises this +// projection makes. +func toolCallFromEvent(e events.Event) edastore.ToolCall { + tool, _ := e.Data["tool"].(string) + failed, _ := e.Data["failed"].(bool) + tc := edastore.ToolCall{ + TaskID: e.TaskID, + Attempt: e.Attempt, + Tool: tool, + CalledAt: e.At, + } + if failed { + tc.Error = e.Detail + } else { + tc.Result = e.Detail + } + return tc +} diff --git a/internal/app/archied/state_store_toolcalls_test.go b/internal/app/archied/state_store_toolcalls_test.go new file mode 100644 index 000000000..1371f7d55 --- /dev/null +++ b/internal/app/archied/state_store_toolcalls_test.go @@ -0,0 +1,192 @@ +package archied + +import ( + "bytes" + "context" + "errors" + "log/slog" + "strings" + "testing" + "time" + + "github.com/samcharles93/archie-core/internal/events" + "github.com/samcharles93/archie-core/internal/infrastructure/edastore" + "github.com/samcharles93/archie-core/internal/infrastructure/staterpc" + "github.com/samcharles93/archie-core/internal/store" +) + +func toolCallEvent(taskID int64, attempt int) events.Event { + return events.Event{ + Kind: events.KindToolCall, + TaskID: taskID, + Attempt: attempt, + Stage: "implement", + Detail: "all checks passed", + Data: map[string]any{"tool": "shell", "failed": false}, + At: time.Now().UTC(), + Workflow: "tdd", + } +} + +// TestInsertEventProjectsToolCall pins the projection (test a): one tool_call +// event through the decorated store persists the event AND exactly one +// tool_calls row, with the row's fields taken from the event -- the tool name +// from data, the output summary in result for a succeeded call. +func TestInsertEventProjectsToolCall(t *testing.T) { + st := store.OpenTest(t) + eda := edastore.OpenTest(t) + ts := newToolCallProjectingTaskStore(st, eda, slog.Default()) + + id, err := ts.InsertEvent(t.Context(), toolCallEvent(42, 2)) + if err != nil { + t.Fatalf("InsertEvent() error = %v", err) + } + if id == 0 { + t.Fatal("InsertEvent() returned id 0; the event itself must still persist") + } + + rows, err := eda.TaskToolCalls(t.Context(), 42) + if err != nil { + t.Fatalf("TaskToolCalls() error = %v", err) + } + if len(rows) != 1 { + t.Fatalf("TaskToolCalls() returned %d rows, want exactly one per tool_call event: %+v", len(rows), rows) + } + row := rows[0] + if row.TaskID != 42 || row.Attempt != 2 { + t.Errorf("row task/attempt = %d/%d, want 42/2", row.TaskID, row.Attempt) + } + if row.Tool != "shell" { + t.Errorf("row tool = %q, want \"shell\" from the event's data", row.Tool) + } + if row.Result != "all checks passed" { + t.Errorf("row result = %q, want the event detail of a succeeded call", row.Result) + } + if row.Error != "" { + t.Errorf("row error = %q, want empty for a succeeded call", row.Error) + } + if row.CalledAt.IsZero() { + t.Error("row called_at is zero; it must carry the event time") + } +} + +// TestInsertEventProjectsFailedToolCall pins the failure column: a tool_call +// event whose call failed stores its detail under error, not result, so the +// transcript can distinguish the two without re-parsing text. +func TestInsertEventProjectsFailedToolCall(t *testing.T) { + st := store.OpenTest(t) + eda := edastore.OpenTest(t) + ts := newToolCallProjectingTaskStore(st, eda, slog.Default()) + + e := toolCallEvent(42, 2) + e.Detail = "error: no such file" + e.Data = map[string]any{"tool": "read_file", "failed": true} + if _, err := ts.InsertEvent(t.Context(), e); err != nil { + t.Fatalf("InsertEvent() error = %v", err) + } + + rows, err := eda.TaskToolCalls(t.Context(), 42) + if err != nil { + t.Fatalf("TaskToolCalls() error = %v", err) + } + if len(rows) != 1 { + t.Fatalf("TaskToolCalls() returned %d rows, want 1", len(rows)) + } + if rows[0].Error != "error: no such file" { + t.Errorf("row error = %q, want the failed call's detail", rows[0].Error) + } + if rows[0].Result != "" { + t.Errorf("row result = %q, want empty for a failed call", rows[0].Result) + } +} + +// TestInsertEventLeavesOtherKindsUnprojected pins the kind gate: only +// tool_call events reach the collection; the task timeline's other kinds must +// not grow tool_calls rows. +func TestInsertEventLeavesOtherKindsUnprojected(t *testing.T) { + st := store.OpenTest(t) + eda := edastore.OpenTest(t) + ts := newToolCallProjectingTaskStore(st, eda, slog.Default()) + + if _, err := ts.InsertEvent(t.Context(), events.Event{ + Kind: events.KindStageFinish, TaskID: 42, Attempt: 2, + Data: map[string]any{"duration_ms": 12.0}, + }); err != nil { + t.Fatalf("InsertEvent() error = %v", err) + } + + rows, err := eda.TaskToolCalls(t.Context(), 42) + if err != nil { + t.Fatalf("TaskToolCalls() error = %v", err) + } + if len(rows) != 0 { + t.Fatalf("TaskToolCalls() returned %d rows for a stage_finish event, want 0", len(rows)) + } +} + +// failingToolCallWriter stands in for a broken PocketBase side, so the +// never-returned-failure contract can be tested without sabotaging the real +// store. +type failingToolCallWriter struct{ err error } + +func (f failingToolCallWriter) InsertToolCall(context.Context, edastore.ToolCall) error { + return f.err +} + +// TestInsertEventSucceedsWhenProjectionFails pins the load-bearing half of the +// contract: a tool_calls write failure is never returned from InsertEvent -- +// event persistence is the contract, the projection is observability -- and +// the failure is recorded, not swallowed silently. +func TestInsertEventSucceedsWhenProjectionFails(t *testing.T) { + st := store.OpenTest(t) + var buf bytes.Buffer + ts := newToolCallProjectingTaskStore(st, failingToolCallWriter{ + err: errors.New("disk on fire"), + }, slog.New(slog.NewTextHandler(&buf, nil))) + + id, err := ts.InsertEvent(t.Context(), toolCallEvent(42, 2)) + if err != nil { + t.Fatalf("InsertEvent() error = %v; a projection failure must never fail the event write", err) + } + if id == 0 { + t.Fatal("InsertEvent() returned id 0; the event must still persist") + } + if got := ts.toolCallProjectionFailures(); got != 1 { + t.Errorf("toolCallProjectionFailures() = %d, want 1: the failure must be counted, not swallowed", got) + } + if !strings.Contains(buf.String(), "disk on fire") { + t.Errorf("log output %q does not carry the projection failure; it was swallowed silently", buf.String()) + } + // The event row is what the whole surface exists for -- it must be there. + persisted, err := st.TaskEvents(t.Context(), 42) + if err != nil { + t.Fatalf("TaskEvents() error = %v", err) + } + if len(persisted) != 1 { + t.Fatalf("TaskEvents() returned %d events, want 1: the event persisted despite the projection failure", len(persisted)) + } +} + +// TestStateStoreDepsProjectToolCalls pins the wire-in: with both stores open, +// the composition serves Tasks as the projecting decorator; with no +// event-capture store, Tasks is passed through untouched rather than wrapped +// around a nil writer. +func TestStateStoreDepsProjectToolCalls(t *testing.T) { + st := store.OpenTest(t) + eda := edastore.OpenTest(t) + + b := newBootstrap() + b.st = st + b.eda = eda + deps := b.stateStoreDeps(&staterpc.TaskGrants{}) + if _, ok := deps.Tasks.(*toolCallProjectingTaskStore); !ok { + t.Fatalf("deps.Tasks = %T, want the tool_call projecting decorator", deps.Tasks) + } + + plain := newBootstrap() + plain.st = st + plainDeps := plain.stateStoreDeps(&staterpc.TaskGrants{}) + if _, ok := plainDeps.Tasks.(*toolCallProjectingTaskStore); ok { + t.Fatal("deps.Tasks is decorated with no event-capture store on the boot; the projection would write through a nil writer") + } +} diff --git a/internal/infrastructure/edastore/tool_calls.go b/internal/infrastructure/edastore/tool_calls.go new file mode 100644 index 000000000..7a72f1e30 --- /dev/null +++ b/internal/infrastructure/edastore/tool_calls.go @@ -0,0 +1,98 @@ +package edastore + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/pocketbase/pocketbase/core" +) + +// ToolCall is one completed tool invocation during an agent stage run, as the +// tool_call event projection stores it. The projection adapts the event to the +// existing collection (schema.go): a tool's output summary goes in result, a +// failure's detail in error. The args and duration_ms fields the schema also +// carries are left unwritten -- the tool_call event reports neither -- and a +// future producer that has them can start filling them without a schema +// change. +type ToolCall struct { + ID string + TaskID int64 + Attempt int + Tool string + Result string + Error string + CalledAt time.Time +} + +// InsertToolCall appends one row to the tool_calls collection. There is no +// idempotency here, deliberately: the collection is an append-only transcript +// and the schema carries no unique index (the SQLite predecessor had none -- +// tool calls lived only in the on-disk task log), so one event is one row. +// +// The notify hook stays silent, matching the task-dispatch tables rather than +// the operator-editable ones: tool calls are high-frequency telemetry, not +// dashboard activity. +func (s *Store) InsertToolCall(_ context.Context, tc ToolCall) error { + collection, err := s.app.FindCollectionByNameOrId(CollToolCalls) + if err != nil { + return err + } + r := core.NewRecord(collection) + r.Set("task_id", tc.TaskID) + r.Set("attempt", tc.Attempt) + r.Set("tool", tc.Tool) + if tc.Result != "" { + r.Set("result", tc.Result) + } + if tc.Error != "" { + r.Set("error", tc.Error) + } + if !tc.CalledAt.IsZero() { + // called_at is an autodate field, but an explicitly set date is kept: + // the row carries the time the tool call happened, not the time it + // was projected. + r.Set("called_at", tc.CalledAt) + } + if err := s.app.Save(r); err != nil { + return fmt.Errorf("edastore: insert tool call: %w", err) + } + return nil +} + +// TaskToolCalls returns one task's tool calls in call order. +func (s *Store) TaskToolCalls(_ context.Context, taskID int64) ([]ToolCall, error) { + records, err := s.app.FindRecordsByFilter(CollToolCalls, "task_id = {:task}", "+called_at", 0, 0, + map[string]any{"task": taskID}) + if err != nil { + return nil, fmt.Errorf("edastore: list tool calls: %w", err) + } + out := make([]ToolCall, 0, len(records)) + for _, r := range records { + out = append(out, ToolCall{ + ID: r.Id, + TaskID: r.GetInt64("task_id"), + Attempt: r.GetInt("attempt"), + Tool: r.GetString("tool"), + Result: jsonTextToString(r.GetString("result")), + Error: r.GetString("error"), + CalledAt: r.GetDateTime("called_at").Time(), + }) + } + return out, nil +} + +// jsonTextToString unwraps the JSON string a JSONField stores a plain text +// summary under, and falls back to the raw text for anything that is not a +// JSON string (e.g. an object a future producer stored directly). +func jsonTextToString(raw string) string { + if raw == "" { + return "" + } + var decoded string + if err := json.Unmarshal([]byte(raw), &decoded); err != nil { + return raw + } + return decoded +} diff --git a/internal/infrastructure/edastore/tool_calls_test.go b/internal/infrastructure/edastore/tool_calls_test.go new file mode 100644 index 000000000..d845c977a --- /dev/null +++ b/internal/infrastructure/edastore/tool_calls_test.go @@ -0,0 +1,102 @@ +package edastore_test + +import ( + "testing" + "time" + + "github.com/samcharles93/archie-core/internal/infrastructure/edastore" +) + +func insertToolCall(t *testing.T, s *edastore.Store, tc edastore.ToolCall) { + t.Helper() + if err := s.InsertToolCall(t.Context(), tc); err != nil { + t.Fatalf("InsertToolCall() error = %v", err) + } +} + +// TestInsertToolCallRoundTrip pins the write/read pair on the tool_calls +// collection: a row written with the fields the tool_call projection provides +// reads back unchanged, and the read filters on task_id so one task's calls +// never surface in another's. +func TestInsertToolCallRoundTrip(t *testing.T) { + s := newStore(t) + at := time.Date(2026, 9, 23, 1, 2, 3, 0, time.UTC) + + insertToolCall(t, s, edastore.ToolCall{ + TaskID: 42, + Attempt: 2, + Tool: "shell", + Result: "all checks passed", + CalledAt: at, + }) + insertToolCall(t, s, edastore.ToolCall{ + TaskID: 42, + Attempt: 2, + Tool: "read_file", + Error: "no such file", + CalledAt: at.Add(time.Second), + }) + // A different task's call must not leak into the read below. + insertToolCall(t, s, edastore.ToolCall{ + TaskID: 7, + Attempt: 1, + Tool: "shell", + Result: "other task's call", + CalledAt: at.Add(2 * time.Second), + }) + + got, err := s.TaskToolCalls(t.Context(), 42) + if err != nil { + t.Fatalf("TaskToolCalls() error = %v", err) + } + if len(got) != 2 { + t.Fatalf("TaskToolCalls() returned %d rows for task 42, want 2: %+v", len(got), got) + } + + first, second := got[0], got[1] + // Chronological order: the earlier call first. + if first.Tool != "shell" || second.Tool != "read_file" { + t.Errorf("order = [%q, %q], want [shell, read_file] by called_at", first.Tool, second.Tool) + } + if first.TaskID != 42 || first.Attempt != 2 { + t.Errorf("first row task/attempt = %d/%d, want 42/2", first.TaskID, first.Attempt) + } + if first.Result != "all checks passed" { + t.Errorf("first row result = %q, want the stored summary without JSON quoting", first.Result) + } + if first.Error != "" { + t.Errorf("first row error = %q, want empty: a succeeded call is not an error", first.Error) + } + if second.Error != "no such file" { + t.Errorf("second row error = %q, want the stored failure detail", second.Error) + } + if second.Result != "" { + t.Errorf("second row result = %q, want empty: a failed call has no result", second.Result) + } + if first.CalledAt.IsZero() { + t.Error("first row called_at is zero; the call time must round-trip") + } +} + +// TestInsertToolCallIsAnAppendOnlyTranscript pins that the collection is one +// row per tool call, not an upsert: repeated calls with the same (task, +// attempt, tool) are distinct transcript entries. There is no unique index and +// there must not be one -- the SQLite predecessor had none (tool calls lived +// only in the on-disk task log), so the schema's non-unique (task_id, attempt) +// index stays the only one. +func TestInsertToolCallIsAnAppendOnlyTranscript(t *testing.T) { + s := newStore(t) + at := time.Date(2026, 9, 23, 1, 2, 3, 0, time.UTC) + row := edastore.ToolCall{TaskID: 42, Attempt: 3, Tool: "shell", Result: "ok", CalledAt: at} + + insertToolCall(t, s, row) + insertToolCall(t, s, row) + + got, err := s.TaskToolCalls(t.Context(), 42) + if err != nil { + t.Fatalf("TaskToolCalls() error = %v", err) + } + if len(got) != 2 { + t.Fatalf("TaskToolCalls() returned %d rows, want 2: the transcript is append-only, not a dedup ledger", len(got)) + } +} From 2ef281086c9f540b1a90d3883898b7c58716a4e2 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Wed, 23 Sep 2026 03:33:32 +1000 Subject: [PATCH 20/20] chore(fmt): adopt go fix's range-over-int in two loops task check's fmt pass rewrites both, so the committed tree was not fmt-canonical. Adopted verbatim, per the repository's rule that formatting changes from the configured formatter are law. --- internal/infrastructure/artifactsync/sender.go | 2 +- internal/store/store_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/infrastructure/artifactsync/sender.go b/internal/infrastructure/artifactsync/sender.go index c3fee3ae4..b46d0f3c8 100644 --- a/internal/infrastructure/artifactsync/sender.go +++ b/internal/infrastructure/artifactsync/sender.go @@ -112,7 +112,7 @@ func (c *Client) Publish(ctx context.Context, a Artifact) (string, error) { const attempts = 4 // the initial attempt plus three retries var lastErr error - for attempt := 0; attempt < attempts; attempt++ { + for attempt := range attempts { if attempt > 0 { select { case <-ctx.Done(): diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 6d8425cd7..aa3e770f5 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -553,7 +553,7 @@ func TestEventsSincePagesIdenticalTimestampsWithoutSkipOrRepeat(t *testing.T) { at := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) const total = 50 inserted := make(map[int64]string, total) - for i := 0; i < total; i++ { + for i := range total { detail := fmt.Sprintf("event-%d", i) id, err := s.InsertEvent(ctx, events.Event{Kind: "log", At: at, Detail: detail}) if err != nil {