Files
archie/internal/agentexec/harness_runner.go
T
sam 0f6448c048
deploy / plan (push) Failing after 1s
deploy / gate (push) Skipped
deploy / archied-image (push) Skipped
deploy / agent-image (push) Skipped
deploy / dist-zip (push) Skipped
refactor(agentexec): delete unused runner injection seams, ServeCaptureMCP, an unreachable approval branch and orphan fixtures
2026-11-09 01:43:53 +11:00

551 lines
18 KiB
Go

package agentexec
import (
"bufio"
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"maps"
"os"
"os/exec"
"path/filepath"
"slices"
"strings"
"time"
git "github.com/go-git/go-git/v6"
"github.com/go-git/go-git/v6/plumbing"
"github.com/samcharles93/archie-core/internal/domain/agentrun"
)
// Harness outcomes beyond StatusPassed. The strings match the built-in
// loop's, so a stage reads the same whichever runner ran it.
const (
StatusParked = "parked"
StopTimedOut = "timed_out"
StopPolicy = "policy_violation"
StopGate = "gate_failed"
StopExited = "harness_exited"
)
const (
promptPlaceholder = "{{.Prompt}}"
sessionPlaceholder = "{{.SessionID}}"
stderrTailBytes = 2000
killGrace = 2 * time.Second
)
// HarnessOutput reads one harness invocation's standard output. An adapter
// knows its CLI's stream format; it reports tool calls as they complete and
// the session to resume.
type HarnessOutput interface {
Line(line []byte, report agentrun.Reporter)
SessionID() string
Usage() agentrun.Usage
}
// HarnessRunner runs a stage on an external coding-agent CLI. Archie's
// guarantees do not depend on the CLI obeying its prompt: the runner checks
// the worktree and the repository's refs and configuration after every
// invocation, runs the gate itself, and owns the process's lifetime.
type HarnessRunner struct{}
// NewHarnessRunner returns a runner that reads each invocation with the
// adapter the request names.
func NewHarnessRunner() *HarnessRunner { return &HarnessRunner{} }
func (r *HarnessRunner) Run(ctx context.Context, workspace string, req agentrun.Request, report agentrun.Reporter) (agentrun.Result, error) {
if err := validateHarness(req); err != nil {
return agentrun.Result{}, err
}
if err := ownTreeForHarness(req.Harness.User, workspace); err != nil {
return agentrun.Result{}, fmt.Errorf("give the worktree to the harness user: %w", err)
}
mcpConfig, capturesPath, cleanup, err := r.prepareCaptures(req)
if err != nil {
return agentrun.Result{}, err
}
defer cleanup()
res, err := r.run(ctx, workspace, req, mcpConfig, report)
if capturesPath != "" {
if res.Status == agentrun.StatusPassed {
if _, readyErr := os.Stat(capturesPath + ".ready"); readyErr != nil {
return park(res, StopExited, "harness did not initialize Archie MCP; check CLI MCP support and startup errors"), err
}
}
captures, readErr := readCaptures(capturesPath, req.CaptureTools)
if readErr != nil {
return res, errors.Join(err, fmt.Errorf("read captures: %w", readErr))
}
res.Captures = captures
}
return res, err
}
func (r *HarnessRunner) run(ctx context.Context, workspace string, req agentrun.Request, mcpConfig string, report agentrun.Reporter) (agentrun.Result, error) {
res := agentrun.Result{Version: agentrun.ProtocolVersion, TaskID: req.TaskID, Attempt: req.Attempt, Stage: req.Stage}
before, err := snapshotRepo(workspace)
if err != nil {
return agentrun.Result{}, fmt.Errorf("snapshot workspace before harness: %w", err)
}
runCtx := ctx
if req.Budget.WallClock > 0 {
var cancel context.CancelFunc
runCtx, cancel = context.WithTimeout(ctx, req.Budget.WallClock)
defer cancel()
}
maxIterations := 1
if len(req.Harness.Resume) > 0 || len(req.Harness.Continue) > 0 {
maxIterations = max(1, req.Gate.MaxConsecutiveFailures)
}
prompt := harnessPrompt(workspace, req)
verb := req.Harness.Prompt
session := ""
for {
res.Iterations++
out := r.output(req.Harness.Adapter)
exitErr := invoke(runCtx, workspace, req.Harness, harnessArgv(req.Harness, verb, prompt, session, mcpConfig), out, report)
if id := out.SessionID(); id != "" {
session = id
}
res.Usage = addUsage(res.Usage, out.Usage())
res.TokensUsed = res.Usage.TotalTokens
if ctx.Err() != nil {
return res, ctx.Err()
}
res, stopped, err := settle(runCtx, workspace, before, req, res, exitErr)
if err != nil || stopped {
return res, err
}
gateOutput, gateErr := runHarnessGate(runCtx, req.Harness, req.Gate.Commands, workspace)
if gateErr == nil {
res.Status = agentrun.StatusPassed
return res, nil
}
if runCtx.Err() != nil {
return park(res, StopTimedOut, "harness exceeded its wall-clock budget during the gate"), nil
}
if res.Iterations >= maxIterations {
return park(res, StopGate, gateOutput), nil
}
verb = resumeVerb(req.Harness, session)
if len(verb) == 0 {
return park(res, StopGate, gateOutput), nil
}
prompt = "The quality gate failed. Fix the cause, then stop.\n\n" + gateOutput
}
}
// settle checks one finished invocation. Guarantees come first: a policy
// violation parks the step even when the invocation also timed out or
// failed, because what the harness did matters more than how it ended.
func settle(runCtx context.Context, workspace string, before repoState, req agentrun.Request, res agentrun.Result, exitErr error) (agentrun.Result, bool, error) {
after, err := snapshotRepo(workspace)
if err != nil {
return res, true, fmt.Errorf("snapshot workspace after harness: %w", err)
}
res.Changes = changedPaths(before, after)
// An exhausted budget is an outcome of the step, not a runner error.
outOfTime := errors.Is(runCtx.Err(), context.DeadlineExceeded)
switch violation := policyViolation(before, after, res.Changes, req); {
case violation != "":
return park(res, StopPolicy, violation), true, nil
case outOfTime:
return park(res, StopTimedOut, "harness exceeded its wall-clock budget"), true, nil
case exitErr != nil:
return park(res, StopExited, exitErr.Error()), true, nil
}
return res, false, nil
}
func (r *HarnessRunner) output(adapter string) HarnessOutput {
if a, ok := LookupHarnessAdapter(adapter); ok {
return a.NewOutput()
}
return silentOutput{}
}
func validateHarness(req agentrun.Request) error {
if err := req.Validate(); err != nil {
return err
}
h := req.Harness
if h == nil || len(h.Launch) == 0 {
return errors.New("harness runner needs a harness spec with a launch command")
}
if !slices.ContainsFunc(h.Prompt, func(s string) bool { return strings.Contains(s, promptPlaceholder) }) {
return fmt.Errorf("harness prompt verb must carry %s, or the mission is never passed", promptPlaceholder)
}
if _, ok := LookupHarnessAdapter(h.Adapter); h.Adapter != "" && !ok {
return fmt.Errorf("harness output adapter %q is unknown", h.Adapter)
}
if req.Gate.MaxConsecutiveFailures > 1 && len(h.Resume) == 0 && len(h.Continue) == 0 {
return errors.New("stage retries its gate but the harness declares no resume or continue verb")
}
if len(h.Resume) > 0 && !slices.ContainsFunc(h.Resume, func(s string) bool { return strings.Contains(s, sessionPlaceholder) }) {
return fmt.Errorf("harness resume verb must carry %s", sessionPlaceholder)
}
if (len(req.CaptureTools) > 0 || h.CentralTools) && !slices.ContainsFunc(h.MCPConfig, func(s string) bool { return strings.Contains(s, mcpConfigPlaceholder) }) {
return errors.New("stage needs Archie tools but the harness registers no MCP server; its results would be lost")
}
return nil
}
// harnessArgv is the launch command, then the verb, then the prompt verb
// when the verb is a resume or continue, with placeholders substituted as
// raw values rather than shell text.
func harnessArgv(h *agentrun.HarnessSpec, verb []string, prompt, session, mcpConfig string) []string {
argv := slices.Clone(h.Launch)
tails := [][]string{verb}
if !slices.Equal(verb, h.Prompt) {
tails = append(tails, h.Prompt)
}
if mcpConfig != "" {
tails = append([][]string{h.MCPConfig}, tails...)
}
for _, tail := range tails {
for _, arg := range tail {
arg = strings.ReplaceAll(arg, promptPlaceholder, prompt)
arg = strings.ReplaceAll(arg, sessionPlaceholder, session)
arg = strings.ReplaceAll(arg, mcpConfigPlaceholder, mcpConfig)
argv = append(argv, arg)
}
}
return argv
}
// resumeVerb continues the session the last invocation reported, falling
// back to the most recent session when the CLI reported none. It returns nil
// when neither is possible: resuming with an empty session ID would start a
// broken command.
func resumeVerb(h *agentrun.HarnessSpec, sessionID string) []string {
if len(h.Resume) > 0 && sessionID != "" {
return slices.Clone(h.Resume)
}
if len(h.Continue) > 0 {
return slices.Clone(h.Continue)
}
return nil
}
// invoke runs one harness invocation in its own process group, feeding
// stdout to out line by line. Cancelling ctx kills the whole group, so a
// CLI's children cannot outlive the step.
func invoke(ctx context.Context, workspace string, h *agentrun.HarnessSpec, argv []string, out HarnessOutput, report agentrun.Reporter) error {
cmd := exec.CommandContext(ctx, argv[0], argv[1:]...)
cmd.Dir = workspace
cmd.WaitDelay = killGrace
// A nil Env would inherit the worker's environment and its credentials.
cmd.Env = append([]string{}, h.Env...)
setProcessGroup(cmd)
if err := runAsHarnessUser(cmd, h.User); err != nil {
return err
}
stdout, err := cmd.StdoutPipe()
if err != nil {
return err
}
var stderr tailBuffer
cmd.Stderr = io.MultiWriter(&stderr, os.Stderr)
if err := cmd.Start(); err != nil {
return fmt.Errorf("start harness: %w", err)
}
// Keep the raw harness activity visible in the container's live output.
output := io.TeeReader(stdout, os.Stdout)
scanner := bufio.NewScanner(output)
scanner.Buffer(make([]byte, 64*1024), 16*1024*1024)
for scanner.Scan() {
out.Line(scanner.Bytes(), report)
}
if err := scanner.Err(); err != nil {
// Stop the CLI rather than leave it writing into a pipe nobody parses.
_ = cmd.Cancel()
_, _ = io.Copy(io.Discard, output)
_ = cmd.Wait()
return fmt.Errorf("read harness output: %w", err)
}
waitErr := cmd.Wait()
reapGroup(cmd)
if err := waitErr; err != nil {
if ctx.Err() != nil {
return ctx.Err()
}
return fmt.Errorf("harness exited: %w: %s", err, strings.TrimSpace(stderr.String()))
}
return nil
}
func harnessPrompt(workspace string, req agentrun.Request) string {
var b strings.Builder
b.WriteString(req.Mission)
b.WriteString("\n\n## Rules\n\n")
b.WriteString(ProjectScopedRules(workspace, req.ExtraRules))
b.WriteString("\nDo not run git commands that change history, refs or configuration. Archie commits your changes.")
if req.ReadOnly {
b.WriteString("\nThis stage is read-only. Do not create, modify or delete any file.")
}
if patterns := slices.Concat(req.Protection.Suffixes, req.Protection.Globs); len(patterns) > 0 {
b.WriteString("\nDo not modify files matching: " + strings.Join(patterns, ", "))
}
if len(req.Gate.Commands) > 0 {
b.WriteString("\nYour work must pass these commands:")
for _, c := range req.Gate.Commands {
b.WriteString("\n- " + strings.Join(c.Argv, " "))
}
}
if strings.TrimSpace(req.Notes) != "" {
b.WriteString("\n\n## Notes\n\n" + req.Notes)
}
return b.String()
}
// runHarnessGate runs the gate commands confined like the harness: they
// execute code the harness just wrote, so they get its user, its environment
// and its process group rather than the worker's.
func runHarnessGate(ctx context.Context, h *agentrun.HarnessSpec, commands []agentrun.Command, workspace string) (string, error) {
for _, c := range commands {
if len(c.Argv) == 0 {
continue
}
cmd := exec.CommandContext(ctx, c.Argv[0], c.Argv[1:]...)
cmd.Dir = workspace
cmd.WaitDelay = killGrace
cmd.Env = append([]string{}, h.Env...)
setProcessGroup(cmd)
if err := runAsHarnessUser(cmd, h.User); err != nil {
return "", err
}
output, err := cmd.CombinedOutput()
reapGroup(cmd)
switch {
case c.ExpectFailure && err == nil:
return fmt.Sprintf("[%s] expected this command to fail, but it passed:\n%s", c.Name, clipGate(output)),
fmt.Errorf("gate %s: expected failure", c.Name)
case !c.ExpectFailure && err != nil:
return fmt.Sprintf("[%s] %s\n%s", c.Name, strings.Join(c.Argv, " "), clipGate(output)), fmt.Errorf("gate %s: %w", c.Name, err)
}
}
return "", nil
}
func clipGate(out []byte) string {
const limit = 4000
s := strings.TrimSpace(string(out))
if len(s) <= limit {
return s
}
return s[:limit/2] + "\n...[truncated]...\n" + s[len(s)-limit/2:]
}
func park(res agentrun.Result, stop, detail string) agentrun.Result {
res.Status, res.StopReason, res.Detail = StatusParked, stop, detail
return res
}
func addUsage(a, b agentrun.Usage) agentrun.Usage {
return agentrun.Usage{
PromptTokens: a.PromptTokens + b.PromptTokens,
CompletionTokens: a.CompletionTokens + b.CompletionTokens,
TotalTokens: a.TotalTokens + b.TotalTokens,
CachedTokens: a.CachedTokens + b.CachedTokens,
CacheCreationTokens: a.CacheCreationTokens + b.CacheCreationTokens,
}
}
// repoState is what the runner compares across an invocation: every ref
// (HEAD included), the repository configuration, and a content hash of each
// path git reports as not clean.
type repoState struct {
refs map[string]string
config string
dirty map[string]string
}
func snapshotRepo(workspace string) (repoState, error) {
repo, err := git.PlainOpen(workspace)
if err != nil {
return repoState{}, err
}
state := repoState{refs: map[string]string{}, dirty: map[string]string{}}
if head, err := repo.Storer.Reference(plumbing.HEAD); err == nil {
state.refs[plumbing.HEAD.String()] = head.String()
}
iter, err := repo.References()
if err != nil {
return repoState{}, err
}
if err := iter.ForEach(func(ref *plumbing.Reference) error {
state.refs[ref.Name().String()] = ref.String()
return nil
}); err != nil {
return repoState{}, err
}
cfg, err := repo.Config()
if err != nil {
return repoState{}, err
}
raw, err := cfg.Marshal()
if err != nil {
return repoState{}, err
}
state.config = string(raw)
wt, err := repo.Worktree()
if err != nil {
return repoState{}, err
}
status, err := wt.Status()
if err != nil {
return repoState{}, err
}
for path, st := range status {
if st.Worktree == git.Unmodified && st.Staging == git.Unmodified {
continue
}
state.dirty[path] = contentHash(filepath.Join(workspace, path))
}
return state, nil
}
func contentHash(path string) string {
data, err := os.ReadFile(path)
if err != nil {
return "absent"
}
sum := sha256.Sum256(data)
return hex.EncodeToString(sum[:])
}
// changedPaths is every path whose state differs across the invocation. A
// path already dirty and left untouched is not a change of this run.
func changedPaths(before, after repoState) []string {
var changed []string
for path := range mergeKeys(before.dirty, after.dirty) {
if before.dirty[path] != after.dirty[path] {
changed = append(changed, path)
}
}
slices.Sort(changed)
return changed
}
func mergeKeys(a, b map[string]string) map[string]struct{} {
keys := make(map[string]struct{}, len(a)+len(b))
for k := range maps.Keys(a) {
keys[k] = struct{}{}
}
for k := range maps.Keys(b) {
keys[k] = struct{}{}
}
return keys
}
func policyViolation(before, after repoState, changes []string, req agentrun.Request) string {
if !maps.Equal(before.refs, after.refs) {
var moved []string
for name := range mergeKeys(before.refs, after.refs) {
if before.refs[name] != after.refs[name] {
moved = append(moved, name)
}
}
slices.Sort(moved)
return "the harness changed git refs: " + strings.Join(moved, ", ")
}
if before.config != after.config {
return "the harness changed the git configuration"
}
if req.ReadOnly && len(changes) > 0 {
return "the harness wrote in a read-only stage: " + strings.Join(changes, ", ")
}
if protected := ProtectionMatcher(req.Protection, false); protected != nil {
var hit []string
for _, path := range changes {
if protected(path) {
hit = append(hit, path)
}
}
if len(hit) > 0 {
return "the harness modified protected paths: " + strings.Join(hit, ", ")
}
}
return ""
}
type silentOutput struct{}
func (silentOutput) Line([]byte, agentrun.Reporter) {}
func (silentOutput) SessionID() string { return "" }
func (silentOutput) Usage() agentrun.Usage { return agentrun.Usage{} }
// tailBuffer keeps the last stderrTailBytes written to it.
type tailBuffer struct{ buf bytes.Buffer }
func (t *tailBuffer) Write(p []byte) (int, error) {
t.buf.Write(p)
if extra := t.buf.Len() - stderrTailBytes; extra > 0 {
t.buf.Next(extra)
}
return len(p), nil
}
func (t *tailBuffer) String() string { return t.buf.String() }
// prepareCaptures writes the capture tools' spec and an MCP config pointing
// the CLI at archie's MCP server, and creates the captures file the server
// appends to. The directory belongs to the harness user, whose MCP server
// writes into it.
func (r *HarnessRunner) prepareCaptures(req agentrun.Request) (config, captures string, cleanup func(), err error) {
cleanup = func() {}
if len(req.CaptureTools) == 0 && !req.Harness.CentralTools {
return "", "", cleanup, nil
}
dir, err := os.MkdirTemp("", "archie-captures-")
if err != nil {
return "", "", cleanup, err
}
cleanup = func() { _ = os.RemoveAll(dir) }
fail := func(err error) (string, string, func(), error) {
cleanup()
return "", "", func() {}, err
}
spec, err := json.Marshal(harnessMCPSpec{Captures: req.CaptureTools, CentralTools: req.Harness.CentralTools})
if err != nil {
return fail(err)
}
specPath, capturesPath, configPath := filepath.Join(dir, "tools.json"), filepath.Join(dir, "captures.jsonl"), filepath.Join(dir, "mcp.json")
self, err := os.Executable()
if err != nil {
return fail(err)
}
server := map[string]any{
"env": harnessMCPEnvironment(req.Harness.Env),
"command": self,
"args": []string{"mcp", "-spec", specPath, "-captures", capturesPath},
}
cfg, err := json.Marshal(map[string]any{"mcpServers": map[string]any{"archie": server}})
if err != nil {
return fail(err)
}
for path, content := range map[string][]byte{specPath: spec, configPath: cfg, capturesPath: nil} {
if err := os.WriteFile(path, content, 0o600); err != nil {
return fail(err)
}
}
if err := ownForHarness(req.Harness.User, dir, specPath, configPath, capturesPath); err != nil {
return fail(err)
}
registration, err := harnessMCPRegistration(req.Harness.Adapter, dir, configPath, server)
if err != nil {
return fail(err)
}
return registration, capturesPath, cleanup, nil
}