From 8c4bb1ef1d464b46a69cc33f8a8ee2d5300e19fb Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Wed, 22 Jul 2026 02:13:01 +1000 Subject: [PATCH] feat: promote archie-agent to NATS-connected per-task worker Replaces stdin/stdout subprocess transport with NATS JetStream request-reply pattern. The Runner interface and workflow orchestration are unchanged. New NATSRunner (implements Runner): - Publishes AgentRequestMessage to JetStream with X-Archie-Reply header - Blocks on core NATS inbox subscription for response - Same Run(ctx, workspace, req) (Result, error) contract Rewritten archie-agent (cmd/archie-agent): - Long-running NATS-connected process (was 28-line stdin/stdout) - Pull-consumes from ARCHIE_TASKS with archie.agent.> filter - Builds InProcessRunner, executes agent loop, replies to inbox - Graceful shutdown via signal handling Extended internal/nats: - SubjectAgentWildcard, SubjectForAgentRequest, ReplyHeader - Conn() accessor, NewReplyInbox(), PublishMsg() for JetStream - Stream subjects extended to ["archie.task.>", "archie.agent.>"] Config: "nats" added as valid agent.mode alongside inprocess/subprocess Tests: NATSRunner round-trip with embedded NATS server + handler --- .scratch/explore-chunkB-agentexec.md | 329 ++++++++++++++++++++++++++ .scratch/explore-chunkB-nats-reply.md | 322 +++++++++++++++++++++++++ .scratch/explore-chunkB-workflow.md | 188 +++++++++++++++ cmd/archie-agent/main.go | 177 +++++++++++++- cmd/archied/main.go | 10 + config.example.toml | 4 +- internal/agentexec/nats.go | 107 +++++++++ internal/agentexec/nats_test.go | 130 ++++++++++ internal/config/config.go | 4 +- internal/nats/client.go | 25 +- internal/nats/subjects.go | 31 ++- 11 files changed, 1309 insertions(+), 18 deletions(-) create mode 100644 .scratch/explore-chunkB-agentexec.md create mode 100644 .scratch/explore-chunkB-nats-reply.md create mode 100644 .scratch/explore-chunkB-workflow.md create mode 100644 internal/agentexec/nats.go create mode 100644 internal/agentexec/nats_test.go diff --git a/.scratch/explore-chunkB-agentexec.md b/.scratch/explore-chunkB-agentexec.md new file mode 100644 index 000000000..86f4051d1 --- /dev/null +++ b/.scratch/explore-chunkB-agentexec.md @@ -0,0 +1,329 @@ +# agentexec + +Now I have all the information needed. Here is the comprehensive report. + +--- + +## 1. All Fields of Invocation, Request, Result, Response + +All types are defined in `/work/apps/archie-core/internal/agentexec/protocol.go`. + +**Invocation** (lines 139-144): +```go +Version int `json:"version"` +Workspace string `json:"workspace"` +Request Request `json:"request"` +Providers map[string]Provider `json:"providers"` +``` + +**Request** (lines 59-74): +```go +Version int `json:"version"` +TaskID int64 `json:"task_id"` +Attempt int `json:"attempt"` +Stage string `json:"stage"` +Model string `json:"model"` +Mission string `json:"mission"` +ExtraRules string `json:"extra_rules,omitempty"` +ReadOnly bool `json:"read_only,omitempty"` +Budget Budget `json:"budget"` +Gate Gate `json:"gate"` +Preflight []Command `json:"preflight,omitempty"` +Protection Protection `json:"protection"` +Notes string `json:"notes,omitempty"` +CaptureTools []CaptureTool `json:"capture_tools,omitempty"` +``` + +**Result** (lines 97-111): +```go +Version int `json:"version"` +TaskID int64 `json:"task_id"` +Attempt int `json:"attempt"` +Stage string `json:"stage"` +Status string `json:"status"` +StopReason string `json:"stop_reason,omitempty"` +Changes []string `json:"changes,omitempty"` +Iterations int `json:"iterations,omitempty"` +TokensUsed int `json:"tokens_used,omitempty"` +Summary string `json:"summary,omitempty"` +Detail string `json:"detail,omitempty"` +AppendedNotes []string `json:"appended_notes,omitempty"` +Captures map[string][]json.RawMessage `json:"captures,omitempty"` +``` + +**Response** (lines 168-172): +```go +Version int `json:"version"` +Result Result `json:"result"` +Error string `json:"error,omitempty"` +``` + +Supporting types referenced above: + +- **Budget** (lines 31-35): `MaxSteps int`, `MaxTokens int`, `WallClock time.Duration` +- **Gate** (lines 25-28): `Commands []Command`, `MaxConsecutiveFailures int` +- **Command** (lines 18-22): `Name string`, `Argv []string`, `ExpectFailure bool` +- **Protection** (lines 38-41): `Suffixes []string`, `Globs []string` +- **CaptureTool** (lines 45-53): `Name string`, `Description string`, `Parameters json.RawMessage`, `RequiredFields []string`, `NonEmptyStrings []string`, `BooleanFields []string`, `MaxCalls int` +- **Provider** (lines 115-119): `Class string`, `APIKeyEnv string`, `BaseURL string` + +--- + +## 2. ServeOne Step by Step + +Defined in `/work/apps/archie-core/internal/agentexec/worker.go`. + +**`ServeOne` (lines 16-20)** is the public entry point. It delegates to `serveOne` with a factory closure that, given an `Invocation`, returns a runner: + +```go +func ServeOne(ctx context.Context, in io.Reader, out io.Writer, log *slog.Logger) error { + return serveOne(ctx, in, out, func(invocation Invocation) Runner { + return NewInProcessRunner(NewRuntime(invocation.Providers), log) + }) +} +``` + +**`serveOne` (lines 22-43)** does: + +1. **Decode invocation** (line 24): Calls `decodeOne(in, &invocation)` which reads up to `maxProtocolBytes+1` bytes (8 MB + 1) from `in`. Rejects if the value exceeds 8 MB. JSON-decodes into the value (here, `Invocation`). Also checks there is exactly one JSON value (rejects trailing JSON values, line 58-61). + +2. **Validate invocation** (line 27): Calls `invocation.Validate()` which checks: + - `Version == ProtocolVersion` (must be 1) + - `Workspace` is non-empty + - `Providers` map is non-empty, and every entry's `Class` is non-empty and `BaseURL` is parseable without userinfo/query/fragment + - The inner `Request.Validate()`: `Version` must be 1, `TaskID` > 0, `Attempt` > 0, `Stage` non-empty, `Model` non-empty + +3. **Create runner** (line 30): Calls `newRunner(invocation)` -- in ServeOne's case, `NewInProcessRunner(NewRuntime(invocation.Providers), log)`. + +4. **Run agent** (line 34): `runner.Run(ctx, invocation.Workspace, invocation.Request)` returns `result, runErr`. + +5. **Build response** (lines 35-38): Constructs `Response{Version: ProtocolVersion, Result: result}`. If `runErr != nil`, sets `response.Error = runErr.Error()`. + +6. **Encode response to stdout** (lines 39-41): `json.NewEncoder(out).Encode(response)`. Returns error if encoding fails. + +7. **Return nil** on success. + +Key design detail: The `serveOne` generic function separates the wire protocol (stdin/stdout JSON framing) from the runner creation strategy. The factory `func(Invocation) Runner` allows different strategies (in-process vs. subprocess vs. future NATS). Currently `ServeOne` hardcodes the in-process factory, but a future NATS-based archie-agent could call `serveOne` with a different factory or skip it entirely. + +--- + +## 3. SubprocessRunner.Run Step by Step + +Defined in `/work/apps/archie-core/internal/agentexec/subprocess.go`. + +The `SubprocessRunner` struct (lines 17-27): +```go +type SubprocessRunner struct { + Command string + Args []string + Environ []string // daemon's os.Environ() + AdditionalEnv []string // operator-approved env var names + Diagnostics io.Writer // typically os.Stderr + Providers map[string]Provider +} +``` + +**`Run` (lines 29-94)**: + +1. **Extract provider name from model** (lines 30-32): Splits `req.Model` at the first "/". E.g. `"anthropic/claude-sonnet-4"` yields providerName=`"anthropic"`. If no "/" or empty, returns error. + +2. **Look up provider** (lines 34-37): Indexes `r.Providers[providerName]`. Missing returns error. + +3. **Build and validate invocation** (lines 38-42): Creates `Invocation{Version, Workspace, Request, Providers}` with just that one provider entry. Validates. + +4. **Check command** (lines 43-45): `strings.TrimSpace(r.Command)` must be non-empty. + +5. **Marshal invocation** (lines 46-50): `json.Marshal(invocation)` + appends newline. + +6. **Create OS command** (line 52): `exec.CommandContext(ctx, r.Command, r.Args...)`. + +7. **Configure process cancellation** (line 53): `configureProcessCancellation(cmd)` -- on Unix this sets `Setpgid: true` and a `Cancel` function that does `syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)` (see `/work/apps/archie-core/internal/agentexec/subprocess_process_unix.go` lines 15-27). This ensures the worker process AND all its grandchildren die on context cancellation. + +8. **Set stdin** (line 54): `cmd.Stdin = bytes.NewReader(payload)`. + +9. **Set environment** (line 55): `WorkerEnvironment(r.Environ, r.AdditionalEnv, providers)` which (lines 155-162): + - Starts with `DefaultEnvironmentNames()`: `PATH, HOME, TMPDIR, TMP, TEMP, LANG, LC_ALL, SSL_CERT_FILE, SSL_CERT_DIR` + - Appends `r.AdditionalEnv` + - Appends each provider's `APIKeyEnv` + - Calls `SelectEnvironment` which copies only those named variables from `r.Environ`, preserving order. + +10. **Capture stdout and stderr** (lines 56-62): + - Stdout: `cappedBuffer{limit: maxProtocolBytes}` (8 MB). If `r.Diagnostics != nil`, stderr goes to `io.MultiWriter(r.Diagnostics, stderr)` (tee to both the diagnostics writer and the capped buffer). Otherwise just the capped buffer (64 KB limit via `maxDiagnosticBytes`). + +11. **Run the process** (lines 63-71): `cmd.Run()`. On error: + - If `context.Cause(ctx)` is set (cancellation), returns the cause wrapped with any stderr diagnostic. + - Otherwise returns the exec error with stderr output. + +12. **Check stdout truncation** (lines 72-74): If `stdout.truncated` is true, return error. + +13. **Decode response** (lines 75-78): `decodeOne(bytes.NewReader(stdout.Bytes()), &response)` -- same framing as the reader side. + +14. **Validate response version** (lines 79-81): Must match `ProtocolVersion` (1). + +15. **Validate result against request** (lines 82-91): + - If `response.Result.Version != 0`, calls `response.Result.ValidateFor(req)` -- checks Version==1, TaskID/Attempt/Stage identity match, Status non-empty. + - If `response.Error != ""`, returns result with `errors.New(response.Error)`. + - Validates one more time at line 90 for the non-error case. + +16. **Return** (line 93): `response.Result, nil`. + +--- + +## 4. NewRuntime and InProcessRunner + +**`NewRuntime`** in `/work/apps/archie-core/internal/agentexec/runtime.go` (lines 7-23): +- Takes `providers map[string]Provider` (agentexec.Provider type). +- Returns `*runtime.Runtime` (from `github.com/samcharles93/ai-sdk/runtime`). +- If providers is empty, returns `nil`. +- Calls `runtime.RegisterBuiltinClasses()` (global, side-effecting). +- For each entry: creates `runtime.ProviderConfig{ID: name, Class: provider.Class, BaseURL: provider.BaseURL}`. +- If `provider.APIKeyEnv == ""`, sets `Auth = AuthConfig{Type: AuthTypeNone}`, otherwise `Auth = AuthConfig{APIKeyEnv: provider.APIKeyEnv}`. +- Calls `runtime.NewRuntime(runtime.Config{Providers: catalog})` and returns it. + +**`InProcessRunner`** in `/work/apps/archie-core/internal/agentexec/inprocess.go` (lines 29-33): +```go +type InProcessRunner struct { + runtime *runtime.Runtime + log *slog.Logger + run loopFunc // func(context.Context, agentloop.Config) (agentloop.Result, error) +} +``` + +**`NewInProcessRunner`** (lines 35-37): Stores runtime, log, and sets `run = agentloop.Run` (the exported function from the `agentloop` package). + +**`InProcessRunner.Run`** (lines 39-86): +1. Validates `req` via `req.Validate()`. +2. Checks `r.runtime != nil` and `r.run != nil`. +3. Creates `memoryNotes` struct (lines 219-228 of inprocess.go) -- implements a simple `Load`/`Append` interface. Initialized with `req.Notes`, collects `appended` entries. +4. Creates `captures` map: `map[string][]json.RawMessage`. +5. Calls `r.run(ctx, agentloop.Config{...})` with all fields mapped from `req`: + - `Runtime: r.runtime` + - `ModelRef: req.Model` + - `WorkDir: workspace` + - `Mission: req.Mission`, `ExtraRules: req.ExtraRules` + - `Notes: notes` (the memoryNotes) + - `Gate: toAgentGate(req.Gate)` (converts agentexec.Gate -> agentloop.GateConfig) + - `Preflight: toAgentCommands(req.Preflight)` + - `Budget: agentloop.Budget(req.Budget)` (direct cast) + - `ReadOnly: req.ReadOnly` + - `ProtectPaths: protectionMatcher(...)` (closure that checks suffixes/globs) + - `Extra: mergeToolSets(captureToolSet(req.CaptureTools, captures), scriptToolSet(workspace))` + - `Logger: r.logger(req)` (structured with task/attempt/stage/model) +6. Builds `Result` from the returned `agentloop.Result` fields. +7. Checks `context.Cause(ctx)` for cancellation -- if cancelled, returns result with cancellation cause as error. +8. Returns `result, err`. + +--- + +## 5. Provider Types + +There are **two separate `Provider` structs** with identical fields but different packages and tags: + +**`agentexec.Provider`** (`/work/apps/archie-core/internal/agentexec/protocol.go`, lines 115-119): +```go +type Provider struct { + Class string `json:"class"` + APIKeyEnv string `json:"api_key_env,omitempty"` + BaseURL string `json:"base_url,omitempty"` +} +``` +Used for: JSON serialization to the subprocess worker, and as the type consumed by `NewRuntime` and `SubprocessRunner.Providers`. + +**`config.Provider`** (`/work/apps/archie-core/internal/config/config.go`, lines 125-129): +```go +type Provider struct { + Class string `toml:"class"` + APIKeyEnv string `toml:"api_key_env"` + BaseURL string `toml:"base_url"` +} +``` +Used for: TOML deserialization from the config file. + +**Bridge** (`/work/apps/archie-core/cmd/archied/main.go`, lines 182-188): +```go +func executionProviders(cfg config.Config) map[string]agentexec.Provider { + providers := make(map[string]agentexec.Provider, len(cfg.Providers)) + for name, p := range cfg.Providers { + providers[name] = agentexec.Provider{Class: p.Class, APIKeyEnv: p.APIKeyEnv, BaseURL: p.BaseURL} + } + return providers +} +``` +The daemon composition root converts config.Provider to agentexec.Provider by field-wise copy and then passes them to both `NewRuntime` and `SubprocessRunner`. + +--- + +## 6. How drainNATS Interacts with SubprocessRunner + +**Current state: drainNATS only affects task distribution, not agent execution.** + +The flow: + +1. **Daemon.Cycle** (daemon.go lines 94-103): If `d.Nats != nil`, calls `d.drainNATS(ctx)`, else `d.drainSQLite(ctx)`. + +2. **drainNATS** (lines 126-151): Calls `d.Nats.Fetch(ctx)` to get a JetStream message. If found, calls `processNATSTask`. Falls back to `drainSQLite` logic (SQLite `ClaimNext` then `d.process`) if no NATS message. + +3. **processNATSTask** (lines 219-253): Decodes `TaskMessage`, enqueues to SQLite, claims the task, then calls `d.process(ctx, task)` -- the **same** `process` function used by the SQLite path. + +4. **d.process** (lines 472-493): Routes the task to the correct workflow, then calls: + ```go + workflow.Run(ctx, wf, &workflow.TaskContext{ + ... + Agent: d.Agent, // line 488 -- the agentexec.Runner interface + ... + }) + ``` + +5. **workflow.Run** (workflow.go line 145): Iterates stages. For LLM-driven stages, calls `AgentStage.Stage()` (agent.go line 110): + ```go + res, err := tc.Agent.Run(ctx, tc.Dir, req) + ``` + This `Agent` field is the `agentexec.Runner` interface. It was wired in `cmd/archied/main.go` (lines 124-138): + - If `cfg.Agent.Mode == "subprocess"`: creates `&agentexec.SubprocessRunner{...}` with `Command`, `Environ`, `AdditionalEnv`, `Diagnostics`, `Providers`. + - If `cfg.Agent.Mode == "inprocess"`: creates `agentexec.NewInProcessRunner(llm, log)`. + +6. **SubprocessRunner.Run** spawns a fresh `archie-agent` binary per stage, sends JSON on stdin, reads JSON from stdout. + +7. **The daemon's NATS `processNATSTask` does NOT interact with SubprocessRunner directly** -- it only fetches the task message from NATS, enqueues it to SQLite, and claims it. The actual agent execution happens through `d.Agent.Run` which remains either in-process or subprocess regardless of the NATS path. + +--- + +## Design Implications for NATSRunner and NATS-based archie-agent + +**Runner interface** (`/work/apps/archie-core/internal/agentexec/inprocess.go`, line 20-22): +```go +type Runner interface { + Run(ctx context.Context, workspace string, req Request) (Result, error) +} +``` + +A new `NATSRunner` would need to implement this same interface. It would: + +1. Accept a NATS connection, a subject prefix, and the provider map (or the whole Invocation fields). +2. In `Run(ctx, workspace, req)`: + - Build an `Invocation` (same as `SubprocessRunner.Run` does on lines 38-42). + - Publish it to a NATS subject (e.g. `archie.stage.request.`). + - Include a reply subject in the message headers or as part of the payload. + - Wait for the response on the reply subject (with the context's deadline). + - Decode the `Response`, validate, return `Result`. +3. No need for stdin/stdout marshalling or process management. + +**New archie-agent** (`cmd/archie-agent/main.go`): Instead of `ServeOne(ctx, os.Stdin, os.Stdout, log)`, it would: +1. Connect to NATS. +2. Subscribe to a subject like `archie.stage.request.>`. +3. For each message: decode `Invocation` from the message body, call `serveOne`-style logic (validate, build runner, run, build response), publish `Response` back to the reply subject, ack. +4. Could be a long-running daemon with a pool of subscribers. + +**Key integration point**: The daemon composition root (`cmd/archied/main.go` lines 124-138) would gain a third branch: +```go +case "nats": + agentRunner = &agentexec.NATSRunner{ + Conn: natsClient.Conn(), // or a new NATS connection + Subject: "archie.stage.request", + Providers: providers, + } +``` + +And in `config.go` the `Agent.Mode` validation (lines 298-305) would accept `"nats"` in addition to `"inprocess"` and `"subprocess"`. + +**The `serveOne` generic function** in `worker.go` (lines 22-43) is already well-factored: it separates wire protocol from runner creation. A NATS-based archie-agent could reuse `serveOne` by providing a factory that creates an in-process runner, or it could have its own message handler that calls the same `Validate` -> `Runner.Run` -> `Response` logic. \ No newline at end of file diff --git a/.scratch/explore-chunkB-nats-reply.md b/.scratch/explore-chunkB-nats-reply.md new file mode 100644 index 000000000..fbbda9265 --- /dev/null +++ b/.scratch/explore-chunkB-nats-reply.md @@ -0,0 +1,322 @@ +# nats-reply + +Now I have a complete understanding. Here is the comprehensive report. + +--- + +## NATS Agent Communication Reply Pattern -- Design Report + +### Sources Read + +- `/work/apps/archie-core/internal/nats/client.go` -- existing JetStream Client, PublishTask, Fetch, Connect +- `/work/apps/archie-core/internal/nats/subjects.go` -- existing subject constants +- `/work/apps/archie-core/internal/daemon/daemon.go` -- current NATS integration in the daemon +- `/work/apps/archie-core/internal/config/config.go` -- NATSConfig struct +- `/work/apps/archie-core/go.mod` -- nats.go v1.52.0, nats-server/v2 v2.14.3 +- `/work/apps/archie-core/internal/agentexec/inprocess.go` -- Runner interface definition (line 20) +- `/work/apps/archie-core/internal/agentexec/subprocess.go` -- SubprocessRunner.Run (blocks on exec.Command) +- `/work/apps/archie-core/internal/agentexec/protocol.go` -- Request, Result, Invocation, Response types +- `/work/apps/archie-core/internal/workflow/agent.go` -- how AgentStage calls `tc.Agent.Run(ctx, tc.Dir, req)` (line 110) +- `/home/sam/go/pkg/mod/github.com/nats-io/nats.go@v1.52.0/jetstream/publish.go` -- PublishMsg internals +- `/home/sam/go/pkg/mod/github.com/nats-io/nats.go@v1.52.0/jetstream/message.go` -- jetstream.Msg interface (Reply(), Headers(), etc.) +- `/home/sam/go/pkg/mod/github.com/nats-io/nats.go@v1.52.0/context.go` -- RequestMsgWithContext, requestWithContext +- `/home/sam/go/pkg/mod/github.com/nats-io/nats.go@v1.52.0/nats.go` -- Msg struct, NewInbox, createNewRequestAndSend +- `/work/apps/archie-core/.scratch/plan-nats-design.md` -- the original Phase 1 NATS task distribution design document + +--- + +### Key Finding: `PublishMsg` Eats the Reply Field + +The most critical technical finding is in `jetstream/publish.go` (line 229-232) and `context.go` (line 40-84). When you call: + +```go +js.PublishMsg(ctx, &nats.Msg{Subject: subj, Reply: "my.reply", Data: data}) +``` + +The `PublishMsg` implementation calls `js.conn.RequestMsgWithContext(ctx, m)`, which internally calls `createNewRequestAndSend(subj, hdr, data)` (new style) or `oldRequestWithContext` (old style). Both paths create a **fresh inbox** for the JetStream PubAck response and call `nc.publish(subj, respInbox, ...)` -- the reply subject you set on the `nats.Msg` is **completely ignored**. The wire-level `Reply` on the published NATS message is the PubAck inbox, not your custom reply subject. + +There is a second issue: even if the Reply did pass through, it would need to point at an inbox the JetStream server could reach. And `PublishMsgAsync` explicitly rejects messages with `Reply` set (`ErrAsyncPublishReplySubjectSet`, line 336-337 of publish.go). + +**This means: the reply-to address must be carried outside the `nats.Msg.Reply` field -- in custom headers or the message body.** + +--- + +### 1. Subject Naming Pattern + +**Requests** (published by the daemon to JetStream): +``` +archie.agent.. +``` +Examples: `archie.agent.42.planner`, `archie.agent.42.builder`, `archie.agent.17.analyst` + +**Responses** (published by the agent worker back to the daemon): +Use core NATS reply inboxes, generated by `nc.NewInbox()`: +``` +_INBOX. +``` +These are ephemeral, one-shot subjects. No JetStream stream is involved on the response path. + +**Why not a deterministic subject for responses?** The daemon needs to wait for exactly one response from a specific agent execution. An inbox subscription is the NATS-idiomatic way: it creates a unique subject per request, the daemon subscribes to it before publishing, and no other subscriber can accidentally consume the response. + +**New subject constants to add to `internal/nats/subjects.go`:** +```go +const SubjectAgentRequest = "archie.agent." // prefix, full subject is built with task_id and stage + +func SubjectForAgentRequest(taskID int64, stage string) string { + return fmt.Sprintf("archie.agent.%d.%s", taskID, stage) +} +``` + +--- + +### 2. JetStream or Core NATS? + +**Hybrid: JetStream for requests, core NATS for responses.** + +| Aspect | Requests | Responses | +|--------|----------|-----------| +| Transport | JetStream (`js.PublishMsg`) | Core NATS (`nc.Publish`) | +| Persistence | Yes (survives daemon crash) | No (ephemeral) | +| Dedup | Yes (Nats-Msg-Id header) | Not needed | +| Delivery guarantee | At-least-once (consumer retries) | Best-effort (daemon subscribed) | + +**Rationale:** +- The **request** (agentexec.Request) is a large, important payload. If the daemon crashes after publishing but before the response arrives, JetStream retains the message. A restarted daemon (or a different daemon on the same stream) can pick it up. +- The **response** is an ephemeral, one-shot delivery. If the daemon disconnects while waiting (crash, network blip), the response is lost -- but that is acceptable because the agent will eventually time out waiting for a result confirmation and the daemon can re-issue the request. +- Using JetStream for responses would require a second pull consumer and polling -- incompatible with the blocking `Runner.Run` contract. +- The existing `nc.Conn` is already available via `js.Conn()`, so no extra connection is needed. + +--- + +### 3. Correlating Requests with Responses + +Use **reply inbox** correlation -- the standard NATS request-reply pattern: + +1. Daemon calls `nc.NewInbox()` to get a unique reply subject +2. Daemon subscribes to that inbox via `nc.SubscribeSync(inbox)` with `AutoUnsubscribe(1)` +3. Daemon sets the inbox address in a **custom header** `X-Archie-Reply` on the JetStream message +4. Daemon publishes the agent request via `js.PublishMsg` +5. Daemon blocks on `sub.NextMsg(timeout)` waiting for the response +6. Agent worker receives the JetStream message, reads the `X-Archie-Reply` header +7. Agent worker processes the request (may take minutes) +8. Agent worker publishes the result to the reply inbox via `nc.Publish(replySubject, jsonResult)` +9. Daemon receives the response and returns `agentexec.Result` + +**No separate correlation ID is needed.** The reply inbox IS the correlation mechanism. The `agentexec.Response` already carries `TaskID`, `Attempt`, and `Stage` for validation (`ValidateFor` in `agentexec/protocol.go:176-190`), so we get double-checking on the identity of the response without any new identifiers. + +--- + +### 4. Changes to `internal/nats` + +#### 4a. New message types + +Add to `client.go` (or a new `agent.go`): +```go +// AgentRequestMessage is the NATS payload for an agent stage execution request. +type AgentRequestMessage struct { + TaskID int64 `json:"task_id"` + Attempt int `json:"attempt"` + Stage string `json:"stage"` + Workspace string `json:"workspace"` + Request agentexec.Request `json:"request"` + Providers map[string]agentexec.Provider `json:"providers,omitempty"` +} + +// AgentResponseEnvelope is the response payload published to the reply inbox. +type AgentResponseEnvelope struct { + Version int `json:"version"` + Result agentexec.Result `json:"result"` + Error string `json:"error,omitempty"` +} +``` + +#### 4b. New methods on `Client` + +```go +// ReplyHeader is the custom header carrying the daemon's reply inbox. +const ReplyHeader = "X-Archie-Reply" + +// NewReplyInbox creates a unique inbox and returns a sync subscription. +func (c *Client) NewReplyInbox() (*nats.Subscription, error) { + sub, err := c.conn.SubscribeSync(c.conn.NewInbox()) + if err != nil { + return nil, err + } + sub.AutoUnsubscribe(1) // exactly one response expected + return sub, nil +} + +// PublishAgentRequest publishes an agent execution request to JetStream +// with the reply inbox set in the X-Archie-Reply header. +func (c *Client) PublishAgentRequest(ctx context.Context, req agentexec.Request, providers map[string]agentexec.Provider, workspace string, replyInbox string) error { + msg := AgentRequestMessage{ + TaskID: req.TaskID, + Attempt: req.Attempt, + Stage: req.Stage, + Workspace: workspace, + Request: req, + Providers: providers, + } + data, err := json.Marshal(msg) + if err != nil { + return err + } + + subject := SubjectForAgentRequest(req.TaskID, req.Stage) + headers := nats.Header{} + headers.Set(ReplyHeader, replyInbox) + msgID := fmt.Sprintf("%s%d/%d/%s", msgPrefix, req.TaskID, req.Attempt, req.Stage) + headers.Set("Nats-Msg-Id", msgID) + + _, err = c.js.PublishMsg(ctx, &nats.Msg{ + Subject: subject, + Data: data, + Header: headers, + }) + return err +} +``` + +#### 4c. New stream or extend existing + +The existing `ARCHIE_TASKS` stream has subjects `["archie.task.>"]`. Agent requests use `archie.agent..`, which does **not** match `archie.task.>`. + +**Recommendation: Extend the existing stream's subjects** by adding `"archie.agent.>"` to `StreamConfig.Subjects`. This is simpler than creating a second stream and avoids managing two stream lifecycles: + +```go +stream, err := js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{ + Name: streamName, + Subjects: []string{"archie.task.>", "archie.agent.>"}, + Storage: jetstream.FileStorage, + Retention: jetstream.WorkQueuePolicy, + Duplicates: dedupWindow, +}) +``` + +The same WorkQueuePolicy works: when an agent receives and acks the request, it is removed from the stream. If the daemon crashes, the message stays until the agent acks or `MaxDeliver` is reached. + +**Alternative: Create a dedicated `ARCHIE_AGENT` stream.** This would allow different retention/deduplication settings (e.g., a `LimitsPolicy` with a `MaxAge` of 1 hour vs the current WorkQueue). But this adds complexity. Start with the extension approach. + +#### 4d. NATSRunner: bridge between Runner interface and NATS + +Create `/work/apps/archie-core/internal/agentexec/nats.go`: + +```go +type NATSRunner struct { + Nats *nats.Client + Providers map[string]agentexec.Provider + Log *slog.Logger +} + +func (r *NATSRunner) Run(ctx context.Context, workspace string, req agentexec.Request) (agentexec.Result, error) { + if err := req.Validate(); err != nil { + return agentexec.Result{}, err + } + + // 1. Create reply inbox + replySub, err := r.Nats.NewReplyInbox() + if err != nil { + return agentexec.Result{}, fmt.Errorf("nats reply inbox: %w", err) + } + defer replySub.Unsubscribe() + + // 2. Compute timeout from budget + timeout := defaultAgentTimeout + if req.Budget.WallClock > 0 { + timeout = req.Budget.WallClock + } + + // 3. Publish request with reply header + if err := r.Nats.PublishAgentRequest(ctx, req, r.Providers, workspace, replySub.Subject); err != nil { + return agentexec.Result{}, fmt.Errorf("publish agent request: %w", err) + } + + // 4. Wait for reply (blocking, matching SubprocessRunner.Run behavior) + reply, err := replySub.NextMsg(timeout) + if err != nil { + return agentexec.Result{}, fmt.Errorf("agent reply timeout: %w", err) + } + + // 5. Decode response + var envelope AgentResponseEnvelope + if err := json.Unmarshal(reply.Data, &envelope); err != nil { + return agentexec.Result{}, fmt.Errorf("decode agent response: %w", err) + } + if envelope.Error != "" { + return envelope.Result, errors.New(envelope.Error) + } + if err := envelope.Result.ValidateFor(req); err != nil { + return agentexec.Result{}, err + } + return envelope.Result, nil +} +``` + +--- + +### 5. Simplest Possible Reply Pattern + +**Respondent's note: the paragraphs below are the core answer.** + +The simplest pattern that works with the existing JetStream setup: + +1. **Daemon side** (NATSRunner, blocking like SubprocessRunner): + - `sub, _ := nc.SubscribeSync(nc.NewInbox())` -- creates a unique reply subject as a sync subscription + - `sub.AutoUnsubscribe(1)` -- exactly one reply expected + - Publish to JetStream: `js.PublishMsg(ctx, &nats.Msg{Subject: "archie.agent..", Header: {"X-Archie-Reply": [inbox]}, Data: payload})` + - Block: `msg, err := sub.NextMsg(budget.WallClock)` -- returns `agentexec.Result` on success, error on timeout + - Unsubscribe the inbox + +2. **Agent side** (archie-agent worker, or any consumer): + - Receive via JetStream pull consumer + - `replyTo := msg.Headers().Get("X-Archie-Reply")` + - Process the request (the existing agentloop) + - Result: `nc.Publish(replyTo, jsonBytes)` -- a core NATS publish, no JetStream involved + +3. **No additional JetStream consumers, streams, or streams needed.** The existing `nc.Conn` (exposed as `js.Conn()`) handles both directions. + +**NATS API calls involved:** + +| Operation | API Call | Package | +|-----------|----------|---------| +| Create reply inbox | `nc.NewInbox()` / `nc.SubscribeSync(inbox)` | `nats.go` core | +| Publish request | `js.PublishMsg(ctx, &nats.Msg{...})` | `jetstream` | +| Wait for reply | `sub.NextMsg(timeout)` | `nats.go` core | +| Send reply (agent) | `nc.Publish(reply, data)` | `nats.go` core | + +**Total new lines of code in `internal/nats`:** approximately 100-120 lines (two message structs, two new helper functions, one new subject function, and the header constant). + +**Total new lines for `NATSRunner`:** approximately 50-60 lines. + +--- + +### Configuration and Wiring + +The daemon already supports `agent.mode` with values `"inprocess"` and `"subprocess"`. Add `"nats"` as a third mode. + +In `cmd/archied/main.go`, when `cfg.Agent.Mode == "nats"`: +```go +runner := &agentexec.NATSRunner{ + Nats: natsClient, + Providers: agentexec.ProviderConfig(cfg.Providers), + Log: log.With("runner", "nats"), +} +``` + +No changes are needed to the `workflow` package or the daemon's `process` method -- they already consume `tc.Agent.Run(ctx, tc.Dir, req)` via the `agentexec.Runner` interface. + +--- + +### Critical Technical Decision: Reply Subject in Headers + +The `X-Archie-Reply` header approach was chosen over alternatives: + +- **Reply in `nats.Msg.Reply` field**: Does not work -- `PublishMsg` internally uses the Reply field for the JetStream PubAck response and overwrites whatever you set. +- **Reply in message body**: Would require the agent to parse the JSON body before knowing where to respond -- a chicken-and-egg problem if the body is corrupted or the format changes. +- **Correlation ID in headers + deterministic response subject**: Would require provisioning a consumer on the response subject and polling -- breaks the blocking `Runner.Run` contract. + +The reply inbox approach gives us: +- No polling (daemon blocks on sync subscription) +- Automatic correlation (each request gets a unique inbox) +- No consumer lifecycle management for responses +- Matching the existing `SubprocessRunner` blocking contract exactly \ No newline at end of file diff --git a/.scratch/explore-chunkB-workflow.md b/.scratch/explore-chunkB-workflow.md new file mode 100644 index 000000000..52d06ebde --- /dev/null +++ b/.scratch/explore-chunkB-workflow.md @@ -0,0 +1,188 @@ +# workflow + +Here is the full analysis across all five files. + +--- + +## 1. Fields AgentStage reads from TaskContext to build the Request + +The `AgentStage.Stage()` method (lines 47-148 of `internal/workflow/agent.go`) constructs a `agentexec.Request` and reads the following from `TaskContext`: + +| Request field (line) | TaskContext source | Notes | +|---|---|---| +| `Version` (94) | Constant `agentexec.ProtocolVersion` | Not from TaskContext | +| `TaskID` (96) | `tc.Task.ID` | | +| `Attempt` (97) | `tc.Task.Attempt` | | +| `Stage` (98) | `a.Name` (the AgentStage field) | Not from TaskContext | +| `Model` (99) | `tc.Cfg.Models[a.Role]` | Falls back to `tc.Cfg.Models["builder"]` at line 54 if the role key is empty; returns an error at line 57 if still empty | +| `Mission` (100) | `a.Mission(tc)` | The Mission func receives the full `*TaskContext` — implement's plan/build missions read `tc.Task.Title`, `tc.Task.Body`, `tc.Task.Plan`, `tc.Task.IssueNumber`, `tc.Repo.FullName()` | +| `ExtraRules` (101) | `a.ExtraRules` | Static field on AgentStage, not read from TaskContext per se | +| `ReadOnly` (102) | `a.ReadOnly` | Static field | +| `Budget` (103) | `tc.Cfg.Budgets.MaxSteps`, `.MaxTokens`, `.WallClock.Std()` | Overridden by `a.MaxSteps` if > 0 (line 65-67) | +| `Gate` (104) | `a.Gate(tc)` | The Gate function receives TaskContext; implement's build stage calls `GateFromRepo(tc.Repo, tc.Cfg.Budgets)` (agent.go line 152-161) | +| `Preflight` (105) | `tc.Repo.ResolvedPreflight()` | Split into `agentexec.Command` structs (lines 86-92) | +| `Protection` (106) | `tc.Repo.Protect` + `a.ProtectGlobs(tc)` | Combined; set to zero Protection if `a.ReadOnly` is true (line 82-84) | +| `Notes` (107) | `tc.Task.Notes` | | +| `CaptureTools` (108) | `a.CaptureTools(tc)` | The CaptureTools function receives TaskContext | + +At line 110, `tc.Agent.Run(ctx, tc.Dir, req)` is called, which also reads `tc.Dir` (the worktree directory path) and `tc.Agent` (the Runner interface). + +The Mission function for the **plan** stage (implement.go lines 48-64) reads `tc.Task.Plan`, `tc.Task.IssueNumber`, `tc.Task.Title`, `tc.Task.Body`, `tc.Repo.FullName()`. + +The Mission function for the **build** stage (implement.go lines 83-93) reads `tc.Task.IssueNumber`, `tc.Task.Title`, `tc.Task.Body`, `tc.Task.Plan`, `tc.Repo.FullName()`. + +--- + +## 2. What happens after AgentStage.Run returns — Result processing + +The full post-Run sequence is in `agent.go` lines 110-146: + +1. **Line 111-113: Early-return on hard transport error.** If `err != nil && res.Version == 0`, the error is wrapped and returned immediately — the result is unusable (version 0 signals no valid response). + +2. **Line 114-116: Validate.** `res.ValidateFor(req)` checks that the result is consistent with the request (e.g., tool call parameters match what was requested). Returns validation error. + +3. **Lines 117-124: Persist appended notes.** Any entries in `res.AppendedNotes` are appended to `tc.Task.Notes` (prefixed with "- ") and saved via `tc.Store.Update(ctx, tc.Task)`. If the store write fails, the error is returned. + +4. **Lines 125-127: Return on agent-level error.** If `err != nil` (and we've already passed the version-zero check), wraps and returns it as `"agent run: ..."`. This catches cases where the agent loop ran but produced an error. + +5. **Lines 129-130: Accumulate usage.** `tc.Task.TokensUsed += res.TokensUsed; tc.Task.Iterations += res.Iterations`. + +6. **Lines 131-134: Emit event.** `tc.Emit("agent_finish", a.Name, res.Summary, ...)` with status, stop_reason, tokens, iterations, model. + +7. **Lines 136-142: Check status.** If `res.Status != agentexec.StatusPassed`, it returns an error of the form `"agent {status} ({stop_reason}): {detail or summary clipped to 2000 chars}"`. This handles "blocked", "idle", "parked", etc. — any non-passing status causes the stage to fail. + +8. **Lines 143-145: OnResult callback.** If the stage's `OnResult` func is non-nil, it is called with `(tc, res)`. The implement workflow's plan stage OnResult (lines 65-72 of implement.go) sets `tc.Task.Plan = res.Summary` and posts it as a forge comment. The build stage OnResult (lines 94-97 of implement.go) sets `tc.BuildSummary = res.Summary`. + +9. **Line 146: Return nil** if no OnResult or OnResult succeeded. + +--- + +## 3. Full list of stage names in the implement workflow; which call AgentStage + +From `implement.go` lines 38-109, the stages in order: + +| Order | Stage name | Calls AgentStage? | Implementation | +|---|---|---|---| +| 1 | `"prepare"` | No | `StagePrepareWorktree()` — clones repo, checks out branch (steps.go:21-31) | +| 2 | `"repo-stages"` | No | `StageRepoStages()` — runs Yaegi custom stages from `.archie/stages/*.go` (steps.go:84-104) | +| 3 | `"baseline"` | No | `StageBaselineGate()` — verifies the gate commands pass at base commit before any agent work (implement.go:16-30) | +| 4 | `"plan"` | **Yes** | `AgentStage{...}` — read-only planner, max 15 steps, posts the plan as a comment on the issue (implement.go:43-73) | +| 5 | `"build"` | **Yes** | `AgentStage{...}` — builder with gate, no ReadOnly, mission includes the plan, OnResult captures `tc.BuildSummary` (implement.go:75-98) | +| 6 | `"commit-push"` | No | `StageCommitPush(...)` — commits all worktree changes and pushes the branch (steps.go:51-58) | +| 7 | `"custom-gate"` | No | `StageYaegiGate()` — evaluates `.archie/gate.go` for project-specific rules (steps.go:111-151) | +| 8 | `"diff-cap"` | No | `StageDiffCap()` — parks if the diff exceeds configured line cap (steps.go:63-76) | +| 9 | `"open-pr"` | No | `StageOpenPR(...)` — opens the pull request with BuildSummary as body (steps.go:170-174) | + +Two stages call AgentStage: **"plan"** (read-only, no gate, posts plan comment) and **"build"** (read-write, has gate, captures summary). + +--- + +## 4. Every TaskContext.Forge call in steps.go and implement.go + +All invocations of `tc.Forge.*` or `tc.TaskContext.Forge.*`: + +**In `implement.go`:** + +- **Line 68** — `tc.Forge.Comment(context.Background(), tc.Task.Owner, tc.Task.Repo, tc.Task.IssueNumber, body)` — Post the plan as an issue comment (inside the plan stage's `OnResult`). Uses `context.Background()` rather than the stage's context. + +**In `steps.go`:** + +- **Line 160** — `tc.Forge.CreatePR(ctx, t.Owner, t.Repo, title, tc.Branch, tc.Repo.BaseBranch(), body)` — Create the pull request (inside `OpenPR()` helper, called by `StageOpenPR`). + +**In `workflow.go` (not steps.go or implement.go, but part of the workflow engine that uses `tc.Forge`):** + +- **Line 195** — `tc.Forge.SetStateLabel(...)` — Set the state label on the issue when a workflow finishes with an outcome (inside `finish()`). +- **Line 221** — `tc.Forge.SetStateLabel(...)` — Set the "parked" label on the issue (inside `park()`). +- **Line 225** — `tc.Forge.Comment(ctx, t.Owner, t.Repo, t.IssueNumber, body)` — Post the park reason as an issue comment (inside `park()`). + +Total distinct forge operations during workflow execution: **Comment** (plan posting, park notification), **CreatePR** (opening PRs), **SetStateLabel** (three call sites: finish outcome, park, and park again for the label). + +Note: Forge is also used in `daemon.go` outside the workflow engine — `AcceptInvitations`, `VerifyPush`, `React`, `IssuesWithLabel`, `AssignedIssues`, `PRState`, `RepliesAfter`, `CloseIssue`, `SetStateLabel` (for queue/working labels). But in the workflow stages themselves, only the three operations above. + +--- + +## 5. If agent execution moves to NATS: what stays in archied vs. moves to archie-agent + +The central boundary is `tc.Agent.Run(ctx, tc.Dir, req)` at `agent.go:110`. This is the single point where workflow execution crosses from deterministic orchestration into LLM-driven agent execution. + +### Stays in archied (all non-agent orchestration) + +- **Workflow engine** (`workflow.go`): the `Run()` loop that iterates stages, persists task state after each stage, handles outcomes/parking/finish. +- **All deterministic stages** (steps.go, implement.go): + - `StagePrepareWorktree` — git clone and branch checkout via `tc.Trees` + - `StageRepoStages` — Yaegi custom stage discovery and execution + - `StageBaselineGate` — runs shell gate commands at base commit + - `StageCommitPush` — git commit and push via `tc.Trees` + - `StageYaegiGate` — Yaegi gate evaluation + - `StageDiffCap` — diff line counting + - `StageOpenPR` — forge PR creation +- **AgentStage Result processing** (`agent.go` lines 110-146) — validation, note appending, token/iteration accumulation, event emission, OnResult callbacks (plan posting to forge, BuildSummary capture). +- **TaskContext construction** (`daemon.go:481-492`) — the daemon builds TaskContext from store, config, trees, forge, etc. +- **All forge interactions** (`tc.Forge.*`) — commenting on issues, setting labels, creating PRs. +- **Store operations** (`tc.Store.*`) — task persistence, transitions, claims. +- **Task queueing and polling** (`daemon.go`): `poll`, `pollNATS`, `pollSQLite`, `checkWaiting`, `reconcilePRs`, `drainSQLite`, `drainNATS`. + +### Moves to archie-agent (the LLM execution layer) + +- **The actual `tc.Agent.Run(ctx, tc.Dir, req)` call** — the entire agent loop: LLM chat completion, tool execution (read/write files in the worktree, run shell commands, execute gates), budget tracking, stop reason detection. +- **The worktree directory (`tc.Dir`)** and all file I/O within it. The agent's file read/write/edit tools operate on `tc.Dir`. If archie-agent is a separate process, it needs access to the same worktree (either via shared filesystem, or by serializing file operations back to archied, or by archied exposing a filesystem server). +- **Preflight command execution** (lines 86-92 of agent.go) — `tc.Repo.ResolvedPreflight()` commands are baked into the `Request.Preflight` field and executed by the Runner; they would run in archie-agent. +- **Gate execution** (the gate commands in `Request.Gate`) — run by the Runner during the agent loop. + +### What the NATS bridge layer must carry + +The NATS message would need to convey: + +**From archied to archie-agent (the Request):** +- TaskID, Attempt, Stage name, Model reference +- Mission string (already a string — the produced mission text) +- ExtraRules, ReadOnly flag +- Budget (MaxSteps, MaxTokens, WallClock) +- Gate (command list, max failures) +- Preflight (command list) +- Protection (suffixes + globs) +- Notes +- CaptureTools definitions +- **Worktree access** — either the path (if shared filesystem) or the repo coordinates + branch + commit SHA for the agent to clone locally + +**From archie-agent to archied (the Result):** +- Version, Status, StopReason +- Summary, Detail +- TokensUsed, Iterations +- AppendedNotes (string list) +- Full tool call records (for CaptureTools) +- File modifications list (if archied needs to track what changed) + +### Architectural observation about worktree access + +The critical coupling is `tc.Dir` being passed directly to `tc.Agent.Run` (line 110). Currently the agent loop reads and writes files inside that directory. If archie-agent runs as a separate process: +- **Option A (shared filesystem)**: Both archied and archie-agent mount the same worktree directory. ARCHied passes the path, archie-agent writes directly. Simple but requires shared storage. +- **Option B (archied as filesystem proxy)**: Archie-agent sends file read/write/stat requests back to archied over NATS (or a separate channel). Looser coupling but higher latency and complexity. +- **Option C (archied streams the Request, agent clones independently)**: Archie-agent receives the repo/branch/commit via NATS and clones its own worktree copy. This avoids file-routing latency but requires git credentials in the agent process. + +### Summary diagram + +``` +archied (stays) NATS bridge archie-agent (moves) +─────────────────────── ────────── ────────────────────── +Poll → Enqueue → Claim +Build TaskContext + ↓ +workflow.Run() + prepare (git clone) + repo-stages + baseline (gate verify) + plan (AgentStage) ───── Request ────→ ──────→ agent loop (LLM + file I/O) + ←──── Result ──── ←── gate/preflight execution + OnResult: post plan comment + build (AgentStage) ───── Request ────→ ──────→ agent loop (LLM + file I/O) + ←──── Result ──── ←── gate/preflight execution + OnResult: capture BuildSummary + commit-push (git) + custom-gate (yaegi) + diff-cap + open-pr (forge CreatePR) + finish/outcome +``` + +The workflow engine, stage orchestration, all forge/store interactions, and AgentStage result processing stay in archied. Only the `tc.Agent.Run()` call — the LLM agent loop with its file and command execution — moves to archie-agent. \ No newline at end of file diff --git a/cmd/archie-agent/main.go b/cmd/archie-agent/main.go index 95d1e9fca..33f4b050c 100644 --- a/cmd/archie-agent/main.go +++ b/cmd/archie-agent/main.go @@ -1,15 +1,33 @@ -// Command archie-agent executes one unprivileged autonomous agent stage from -// a versioned JSON invocation on stdin and writes one response to stdout. +// Command archie-agent is a long-running NATS-connected worker that executes +// autonomous agent stages. It subscribes to archie.agent.> on the ARCHIE_TASKS +// JetStream stream, runs the agent loop for each received request, and publishes +// the result back to the daemon's reply inbox. package main import ( "context" + "encoding/json" + "flag" + "fmt" "log/slog" "os" "os/signal" "syscall" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" "github.com/samcharles93/archie-core/internal/agentexec" + arnats "github.com/samcharles93/archie-core/internal/nats" +) + +const ( + streamName = "ARCHIE_TASKS" + consumerName = "archie-agent" + dedupWindow = 2 * time.Minute + pollTimeout = 5 * time.Second + ackWait = 30 * time.Minute ) func main() { @@ -17,12 +35,155 @@ func main() { } func run() int { - ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) - defer stop() - log := slog.New(slog.NewJSONHandler(os.Stderr, nil)) - if err := agentexec.ServeOne(ctx, os.Stdin, os.Stdout, log); err != nil { - log.Error("agent worker failed", "err", err) + natsURL := flag.String("nats-url", "", "NATS server URL (required)") + consumer := flag.String("consumer", consumerName, "JetStream consumer name") + flag.Parse() + + if *natsURL == "" { + fmt.Fprintln(os.Stderr, "error: -nats-url is required") + flag.Usage() return 1 } - return 0 + + log := slog.New(slog.NewJSONHandler(os.Stderr, nil)) + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + // Connect to NATS. + nc, err := nats.Connect(*natsURL) + if err != nil { + log.Error("nats connect failed", "err", err) + return 1 + } + defer nc.Close() + log.Info("nats connected", "url", *natsURL) + + // JetStream setup. + js, err := jetstream.New(nc) + if err != nil { + log.Error("jetstream init failed", "err", err) + return 1 + } + + stream, err := js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{ + Name: streamName, + Subjects: []string{"archie.task.>", "archie.agent.>"}, + Storage: jetstream.FileStorage, + Retention: jetstream.WorkQueuePolicy, + Duplicates: dedupWindow, + }) + if err != nil { + log.Error("stream setup failed", "err", err) + return 1 + } + + cons, err := stream.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{ + Name: *consumer, + Durable: *consumer, + FilterSubject: arnats.SubjectAgentWildcard, + AckPolicy: jetstream.AckExplicitPolicy, + MaxDeliver: 3, + AckWait: ackWait, + InactiveThreshold: 24 * time.Hour, + }) + if err != nil { + log.Error("consumer setup failed", "err", err) + return 1 + } + + log.Info("archie-agent ready", "nats", *natsURL, "consumer", *consumer) + + // Main loop: fetch, run, reply, ack. + for { + if ctx.Err() != nil { + log.Info("archie-agent shutting down") + return 0 + } + + batch, err := cons.Fetch(1, jetstream.FetchMaxWait(pollTimeout)) + if err != nil { + if ctx.Err() != nil { + return 0 + } + log.Error("fetch failed", "err", err) + time.Sleep(1 * time.Second) + continue + } + + var msg jetstream.Msg + for m := range batch.Messages() { + if m != nil { + msg = m + break + } + } + if err := batch.Error(); err != nil { + log.Error("fetch batch error", "err", err) + continue + } + if msg == nil { + continue + } + + // Process the message. + if err := handle(ctx, msg, nc, log); err != nil { + log.Error("handle failed", "err", err) + msg.Nak() + continue + } + msg.Ack() + } +} + +func handle(ctx context.Context, msg jetstream.Msg, nc *nats.Conn, log *slog.Logger) error { + // Decode the request. + var req agentexec.AgentRequestMessage + if err := json.Unmarshal(msg.Data(), &req); err != nil { + return fmt.Errorf("decode request: %w", err) + } + + replyTo := msg.Headers().Get(arnats.ReplyHeader) + if replyTo == "" { + return fmt.Errorf("missing %s header", arnats.ReplyHeader) + } + + log.Info("processing stage", + "task", req.TaskID, + "attempt", req.Attempt, + "stage", req.Stage, + "reply_to", replyTo, + ) + + // Build the agent runner and execute. + llm := agentexec.NewRuntime(req.Providers) + if llm == nil { + return fmt.Errorf("no providers configured in request") + } + runner := agentexec.NewInProcessRunner(llm, log) + result, runErr := runner.Run(ctx, req.Workspace, req.Request) + + // Build and publish the response. + resp := agentexec.AgentResponseEnvelope{ + Version: req.Request.Version, + Result: result, + } + if runErr != nil { + resp.Error = runErr.Error() + } + + data, err := json.Marshal(resp) + if err != nil { + return fmt.Errorf("encode response: %w", err) + } + if err := nc.Publish(replyTo, data); err != nil { + return fmt.Errorf("publish response: %w", err) + } + + log.Info("stage complete", + "task", req.TaskID, + "stage", req.Stage, + "status", result.Status, + ) + return nil } diff --git a/cmd/archied/main.go b/cmd/archied/main.go index 4c6581e3e..562d8b861 100644 --- a/cmd/archied/main.go +++ b/cmd/archied/main.go @@ -134,6 +134,16 @@ func run() int { } case "inprocess": agentRunner = agentexec.NewInProcessRunner(llm, log) + case "nats": + if natsClient == nil { + log.Error("agent.mode is nats but [nats] is not configured") + return 1 + } + agentRunner = &agentexec.NATSRunner{ + Nats: natsClient, + Providers: providers, + Log: log, + } } } d := &daemon.Daemon{ diff --git a/config.example.toml b/config.example.toml index 3ab3dc553..f98adfbbb 100644 --- a/config.example.toml +++ b/config.example.toml @@ -37,8 +37,8 @@ api_key_env = "OPENAI_API_KEY" class = "ollama" [agent] -mode = "inprocess" # "subprocess" runs each autonomous stage through archie-agent -command = "archie-agent" # executable path for subprocess mode +mode = "inprocess" # "subprocess" runs each stage through archie-agent (stdin/stdout); "nats" sends stages over NATS +command = "archie-agent" # executable path for subprocess mode (not used in nats mode) # Subprocess mode isolates the protocol and process lifecycle, not OS privileges. # The worker still runs as the daemon user until the Docker sandbox phase lands. # env = ["GOCACHE"] # extra environment variable names forwarded to the worker diff --git a/internal/agentexec/nats.go b/internal/agentexec/nats.go new file mode 100644 index 000000000..144cf96f1 --- /dev/null +++ b/internal/agentexec/nats.go @@ -0,0 +1,107 @@ +package agentexec + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "time" + + natsio "github.com/nats-io/nats.go" + + arnats "github.com/samcharles93/archie-core/internal/nats" +) + +// AgentRequestMessage is the NATS payload for an agent stage execution request. +type AgentRequestMessage struct { + TaskID int64 `json:"task_id"` + Attempt int `json:"attempt"` + Stage string `json:"stage"` + Workspace string `json:"workspace"` + Request Request `json:"request"` + Providers map[string]Provider `json:"providers,omitempty"` +} + +// AgentResponseEnvelope is the response payload published to the reply inbox. +type AgentResponseEnvelope struct { + Version int `json:"version"` + Result Result `json:"result"` + Error string `json:"error,omitempty"` +} + +const defaultAgentTimeout = 30 * time.Minute + +// NATSRunner sends agent stage execution requests over NATS JetStream and +// waits for replies on core NATS inboxes. It implements Runner. +type NATSRunner struct { + Nats *arnats.Client + Providers map[string]Provider + Log *slog.Logger +} + +// Run publishes a stage execution request to JetStream with an X-Archie-Reply +// header, blocks on the sync inbox subscription, and returns the result. +func (r *NATSRunner) Run(ctx context.Context, workspace string, req Request) (Result, error) { + if err := req.Validate(); err != nil { + return Result{}, err + } + + // 1. Create reply inbox with auto-unsubscribe after one message. + replySub, err := r.Nats.NewReplyInbox() + if err != nil { + return Result{}, fmt.Errorf("nats reply inbox: %w", err) + } + defer replySub.Unsubscribe() + + // 2. Compute timeout from budget. + timeout := defaultAgentTimeout + if req.Budget.WallClock > 0 { + timeout = req.Budget.WallClock + } + + // 3. Publish request with reply header. + msg := AgentRequestMessage{ + TaskID: req.TaskID, + Attempt: req.Attempt, + Stage: req.Stage, + Workspace: workspace, + Request: req, + Providers: r.Providers, + } + data, err := json.Marshal(msg) + if err != nil { + return Result{}, err + } + + subject := arnats.SubjectForAgentRequest(req.TaskID, req.Stage) + headers := natsio.Header{} + headers.Set(arnats.ReplyHeader, replySub.Subject) + + if _, err := r.Nats.PublishMsg(ctx, &natsio.Msg{ + Subject: subject, + Data: data, + Header: headers, + }); err != nil { + return Result{}, fmt.Errorf("publish agent request: %w", err) + } + + // 4. Wait for reply (blocking, matching SubprocessRunner.Run behaviour). + reply, err := replySub.NextMsg(timeout) + if err != nil { + return Result{}, fmt.Errorf("agent reply: %w", err) + } + + // 5. Decode response. + var envelope AgentResponseEnvelope + if err := json.Unmarshal(reply.Data, &envelope); err != nil { + return Result{}, fmt.Errorf("decode agent response: %w", err) + } + if envelope.Error != "" { + return envelope.Result, errors.New(envelope.Error) + } + if err := envelope.Result.ValidateFor(req); err != nil { + return envelope.Result, err + } + return envelope.Result, nil +} diff --git a/internal/agentexec/nats_test.go b/internal/agentexec/nats_test.go new file mode 100644 index 000000000..2afdcab32 --- /dev/null +++ b/internal/agentexec/nats_test.go @@ -0,0 +1,130 @@ +package agentexec + +import ( + "context" + "encoding/json" + "log/slog" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/server" + natssrv "github.com/nats-io/nats-server/v2/test" + natsio "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + arnats "github.com/samcharles93/archie-core/internal/nats" +) + +func startEmbeddedNATS(t *testing.T) *server.Server { + t.Helper() + // RunRandClientPortServer doesn't enable JetStream. Enable it separately. + srv := natssrv.RunRandClientPortServer() + t.Cleanup(srv.Shutdown) + if err := srv.EnableJetStream(&server.JetStreamConfig{StoreDir: t.TempDir()}); err != nil { + t.Fatalf("enable jetstream: %v", err) + } + return srv +} + +// handler subscribes to archie.agent.> and replies with a canned result. +func startHandler(t *testing.T, nc *natsio.Conn) { + t.Helper() + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + stream, err := js.CreateOrUpdateStream(context.Background(), jetstream.StreamConfig{ + Name: "ARCHIE_TASKS", + Subjects: []string{"archie.task.>", "archie.agent.>"}, + Storage: jetstream.FileStorage, + Retention: jetstream.WorkQueuePolicy, + }) + if err != nil { + t.Fatal(err) + } + cons, err := stream.CreateOrUpdateConsumer(context.Background(), jetstream.ConsumerConfig{ + Name: "test-agent", + FilterSubject: arnats.SubjectAgentWildcard, + AckPolicy: jetstream.AckExplicitPolicy, + MaxDeliver: 1, + }) + if err != nil { + t.Fatal(err) + } + go func() { + for { + batch, err := cons.Fetch(1, jetstream.FetchMaxWait(5 * time.Second)) + if err != nil { + return + } + for msg := range batch.Messages() { + if msg == nil { + continue + } + var req AgentRequestMessage + json.Unmarshal(msg.Data(), &req) + replyTo := msg.Headers().Get(arnats.ReplyHeader) + resp := AgentResponseEnvelope{ + Version: req.Request.Version, + Result: Result{ + Version: ProtocolVersion, + TaskID: req.TaskID, + Attempt: req.Attempt, + Stage: req.Stage, + Status: StatusPassed, + Summary: "handler-reply", + TokensUsed: 42, + }, + } + data, _ := json.Marshal(resp) + nc.Publish(replyTo, data) + msg.Ack() + } + } + }() +} + +func TestNATSRunnerRoundTrip(t *testing.T) { + srv := startEmbeddedNATS(t) + url := srv.ClientURL() + + ctx := context.Background() + client, err := arnats.Connect(ctx, url, slog.New(slog.DiscardHandler)) + if err != nil { + t.Fatal(err) + } + defer client.Close() + + // Start the handler that simulates archie-agent. + startHandler(t, client.Conn()) + + runner := &NATSRunner{ + Nats: client, + Providers: map[string]Provider{"openai": {Class: "openai", APIKeyEnv: "TEST_KEY"}}, + Log: slog.New(slog.DiscardHandler), + } + + req := Request{ + Version: ProtocolVersion, + TaskID: 42, + Attempt: 1, + Stage: "test", + Model: "openai/gpt", + Mission: "test mission", + Budget: Budget{MaxSteps: 5, MaxTokens: 100}, + } + + result, err := runner.Run(ctx, "/tmp/test-workspace", req) + if err != nil { + t.Fatal(err) + } + if result.Status != StatusPassed { + t.Fatalf("expected passed, got %s", result.Status) + } + if result.Summary != "handler-reply" { + t.Fatalf("expected 'handler-reply', got %q", result.Summary) + } + if result.TokensUsed != 42 { + t.Fatalf("expected 42 tokens, got %d", result.TokensUsed) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 50994473f..0a9266551 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -296,9 +296,9 @@ func Load(path string) (Config, error) { cfg.Agent.Command = "archie-agent" } switch cfg.Agent.Mode { - case "inprocess", "subprocess": + case "inprocess", "subprocess", "nats": default: - return cfg, fmt.Errorf("config: agent.mode %q is invalid (want inprocess or subprocess)", cfg.Agent.Mode) + return cfg, fmt.Errorf("config: agent.mode %q is invalid (want inprocess, subprocess, or nats)", cfg.Agent.Mode) } if cfg.Agent.Mode == "subprocess" && strings.TrimSpace(cfg.Agent.Command) == "" { return cfg, fmt.Errorf("config: agent.command is required in subprocess mode") diff --git a/internal/nats/client.go b/internal/nats/client.go index 920ddf69a..63e2937bb 100644 --- a/internal/nats/client.go +++ b/internal/nats/client.go @@ -54,7 +54,7 @@ func Connect(ctx context.Context, url string, log *slog.Logger) (*Client, error) stream, err := js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{ Name: streamName, - Subjects: []string{"archie.task.>"}, + Subjects: []string{"archie.task.>", "archie.agent.>"}, Storage: jetstream.FileStorage, Retention: jetstream.WorkQueuePolicy, Duplicates: dedupWindow, @@ -81,9 +81,32 @@ func Connect(ctx context.Context, url string, log *slog.Logger) (*Client, error) return &Client{conn: nc, js: js, stream: stream, consumer: consumer, log: log}, nil } +// Conn returns the underlying NATS connection for core NATS operations +// (inbox subscriptions, direct publishes). +func (c *Client) Conn() *nats.Conn { return c.conn } + +// PublishMsg publishes a message to JetStream with optional headers. +func (c *Client) PublishMsg(ctx context.Context, msg *nats.Msg) (*jetstream.PubAck, error) { + return c.js.PublishMsg(ctx, msg) +} + // Close drains and closes the NATS connection. func (c *Client) Close() { c.conn.Close() } +// NewReplyInbox creates a unique inbox subject and returns a synchronous +// subscription configured to auto-unsubscribe after one message. +func (c *Client) NewReplyInbox() (*nats.Subscription, error) { + sub, err := c.conn.SubscribeSync(c.conn.NewInbox()) + if err != nil { + return nil, err + } + if err := sub.AutoUnsubscribe(1); err != nil { + sub.Unsubscribe() + return nil, err + } + return sub, nil +} + // PublishTask publishes a discovered issue to the appropriate NATS subject. // The Nats-Msg-Id header enables JetStream dedup within the dedup window. func (c *Client) PublishTask(ctx context.Context, owner, repo string, number int, title, body, labels string) error { diff --git a/internal/nats/subjects.go b/internal/nats/subjects.go index 155f69c6a..51ecdd9b0 100644 --- a/internal/nats/subjects.go +++ b/internal/nats/subjects.go @@ -1,10 +1,15 @@ -// Package nats manages the NATS JetStream connection for task distribution. -// When configured, the daemon publishes discovered issues to NATS subjects -// and consumes them back via a JetStream pull consumer. When not configured -// (nil client), the existing SQLite ClaimNext flow is used unchanged. +// Package nats manages the NATS JetStream connection for task distribution +// and agent communication. When configured, the daemon publishes discovered +// issues to NATS subjects and consumes them back via a JetStream pull +// consumer. Agent workers consume stage execution requests and reply via +// core NATS inboxes. When not configured (nil client), the existing SQLite +// ClaimNext flow is used unchanged. package nats -import "strings" +import ( + "fmt" + "strings" +) // Task distribution subjects. Each subject encodes the workflow type so // future multi-daemon deployments can filter by workflow. @@ -35,3 +40,19 @@ func SubjectForLabels(labels []string) string { } return SubjectTaskDefault } + +// Agent communication subjects. +const ( + // SubjectAgentWildcard matches all agent stage execution requests. + SubjectAgentWildcard = "archie.agent.>" +) + +// SubjectForAgentRequest returns the JetStream subject for a stage request. +func SubjectForAgentRequest(taskID int64, stage string) string { + return fmt.Sprintf("archie.agent.%d.%s", taskID, stage) +} + +// ReplyHeader is the custom NATS header carrying the daemon's reply inbox. +// JetStream's PublishMsg consumes the Reply field internally for PubAck, so +// the reply address is carried via this header instead. +const ReplyHeader = "X-Archie-Reply"