551 lines
18 KiB
Go
551 lines
18 KiB
Go
package agentexec
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"maps"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"slices"
|
|
"strings"
|
|
"time"
|
|
|
|
git "github.com/go-git/go-git/v6"
|
|
"github.com/go-git/go-git/v6/plumbing"
|
|
|
|
"github.com/samcharles93/archie-core/internal/domain/agentrun"
|
|
)
|
|
|
|
// Harness outcomes beyond StatusPassed. The strings match the built-in
|
|
// loop's, so a stage reads the same whichever runner ran it.
|
|
const (
|
|
StatusParked = "parked"
|
|
StopTimedOut = "timed_out"
|
|
StopPolicy = "policy_violation"
|
|
StopGate = "gate_failed"
|
|
StopExited = "harness_exited"
|
|
)
|
|
|
|
const (
|
|
promptPlaceholder = "{{.Prompt}}"
|
|
sessionPlaceholder = "{{.SessionID}}"
|
|
stderrTailBytes = 2000
|
|
killGrace = 2 * time.Second
|
|
)
|
|
|
|
// HarnessOutput reads one harness invocation's standard output. An adapter
|
|
// knows its CLI's stream format; it reports tool calls as they complete and
|
|
// the session to resume.
|
|
type HarnessOutput interface {
|
|
Line(line []byte, report agentrun.Reporter)
|
|
SessionID() string
|
|
Usage() agentrun.Usage
|
|
}
|
|
|
|
// HarnessRunner runs a stage on an external coding-agent CLI. Archie's
|
|
// guarantees do not depend on the CLI obeying its prompt: the runner checks
|
|
// the worktree and the repository's refs and configuration after every
|
|
// invocation, runs the gate itself, and owns the process's lifetime.
|
|
type HarnessRunner struct{}
|
|
|
|
// NewHarnessRunner returns a runner that reads each invocation with the
|
|
// adapter the request names.
|
|
func NewHarnessRunner() *HarnessRunner { return &HarnessRunner{} }
|
|
|
|
func (r *HarnessRunner) Run(ctx context.Context, workspace string, req agentrun.Request, report agentrun.Reporter) (agentrun.Result, error) {
|
|
if err := validateHarness(req); err != nil {
|
|
return agentrun.Result{}, err
|
|
}
|
|
if err := ownTreeForHarness(req.Harness.User, workspace); err != nil {
|
|
return agentrun.Result{}, fmt.Errorf("give the worktree to the harness user: %w", err)
|
|
}
|
|
mcpConfig, capturesPath, cleanup, err := r.prepareCaptures(req)
|
|
if err != nil {
|
|
return agentrun.Result{}, err
|
|
}
|
|
defer cleanup()
|
|
res, err := r.run(ctx, workspace, req, mcpConfig, report)
|
|
if capturesPath != "" {
|
|
if res.Status == agentrun.StatusPassed {
|
|
if _, readyErr := os.Stat(capturesPath + ".ready"); readyErr != nil {
|
|
return park(res, StopExited, "harness did not initialize Archie MCP; check CLI MCP support and startup errors"), err
|
|
}
|
|
}
|
|
captures, readErr := readCaptures(capturesPath, req.CaptureTools)
|
|
if readErr != nil {
|
|
return res, errors.Join(err, fmt.Errorf("read captures: %w", readErr))
|
|
}
|
|
res.Captures = captures
|
|
}
|
|
return res, err
|
|
}
|
|
|
|
func (r *HarnessRunner) run(ctx context.Context, workspace string, req agentrun.Request, mcpConfig string, report agentrun.Reporter) (agentrun.Result, error) {
|
|
res := agentrun.Result{Version: agentrun.ProtocolVersion, TaskID: req.TaskID, Attempt: req.Attempt, Stage: req.Stage}
|
|
before, err := snapshotRepo(workspace)
|
|
if err != nil {
|
|
return agentrun.Result{}, fmt.Errorf("snapshot workspace before harness: %w", err)
|
|
}
|
|
runCtx := ctx
|
|
if req.Budget.WallClock > 0 {
|
|
var cancel context.CancelFunc
|
|
runCtx, cancel = context.WithTimeout(ctx, req.Budget.WallClock)
|
|
defer cancel()
|
|
}
|
|
|
|
maxIterations := 1
|
|
if len(req.Harness.Resume) > 0 || len(req.Harness.Continue) > 0 {
|
|
maxIterations = max(1, req.Gate.MaxConsecutiveFailures)
|
|
}
|
|
prompt := harnessPrompt(workspace, req)
|
|
verb := req.Harness.Prompt
|
|
session := ""
|
|
for {
|
|
res.Iterations++
|
|
out := r.output(req.Harness.Adapter)
|
|
exitErr := invoke(runCtx, workspace, req.Harness, harnessArgv(req.Harness, verb, prompt, session, mcpConfig), out, report)
|
|
if id := out.SessionID(); id != "" {
|
|
session = id
|
|
}
|
|
res.Usage = addUsage(res.Usage, out.Usage())
|
|
res.TokensUsed = res.Usage.TotalTokens
|
|
if ctx.Err() != nil {
|
|
return res, ctx.Err()
|
|
}
|
|
|
|
res, stopped, err := settle(runCtx, workspace, before, req, res, exitErr)
|
|
if err != nil || stopped {
|
|
return res, err
|
|
}
|
|
|
|
gateOutput, gateErr := runHarnessGate(runCtx, req.Harness, req.Gate.Commands, workspace)
|
|
if gateErr == nil {
|
|
res.Status = agentrun.StatusPassed
|
|
return res, nil
|
|
}
|
|
if runCtx.Err() != nil {
|
|
return park(res, StopTimedOut, "harness exceeded its wall-clock budget during the gate"), nil
|
|
}
|
|
if res.Iterations >= maxIterations {
|
|
return park(res, StopGate, gateOutput), nil
|
|
}
|
|
verb = resumeVerb(req.Harness, session)
|
|
if len(verb) == 0 {
|
|
return park(res, StopGate, gateOutput), nil
|
|
}
|
|
prompt = "The quality gate failed. Fix the cause, then stop.\n\n" + gateOutput
|
|
}
|
|
}
|
|
|
|
// settle checks one finished invocation. Guarantees come first: a policy
|
|
// violation parks the step even when the invocation also timed out or
|
|
// failed, because what the harness did matters more than how it ended.
|
|
func settle(runCtx context.Context, workspace string, before repoState, req agentrun.Request, res agentrun.Result, exitErr error) (agentrun.Result, bool, error) {
|
|
after, err := snapshotRepo(workspace)
|
|
if err != nil {
|
|
return res, true, fmt.Errorf("snapshot workspace after harness: %w", err)
|
|
}
|
|
res.Changes = changedPaths(before, after)
|
|
// An exhausted budget is an outcome of the step, not a runner error.
|
|
outOfTime := errors.Is(runCtx.Err(), context.DeadlineExceeded)
|
|
switch violation := policyViolation(before, after, res.Changes, req); {
|
|
case violation != "":
|
|
return park(res, StopPolicy, violation), true, nil
|
|
case outOfTime:
|
|
return park(res, StopTimedOut, "harness exceeded its wall-clock budget"), true, nil
|
|
case exitErr != nil:
|
|
return park(res, StopExited, exitErr.Error()), true, nil
|
|
}
|
|
return res, false, nil
|
|
}
|
|
|
|
func (r *HarnessRunner) output(adapter string) HarnessOutput {
|
|
if a, ok := LookupHarnessAdapter(adapter); ok {
|
|
return a.NewOutput()
|
|
}
|
|
return silentOutput{}
|
|
}
|
|
|
|
func validateHarness(req agentrun.Request) error {
|
|
if err := req.Validate(); err != nil {
|
|
return err
|
|
}
|
|
h := req.Harness
|
|
if h == nil || len(h.Launch) == 0 {
|
|
return errors.New("harness runner needs a harness spec with a launch command")
|
|
}
|
|
if !slices.ContainsFunc(h.Prompt, func(s string) bool { return strings.Contains(s, promptPlaceholder) }) {
|
|
return fmt.Errorf("harness prompt verb must carry %s, or the mission is never passed", promptPlaceholder)
|
|
}
|
|
if _, ok := LookupHarnessAdapter(h.Adapter); h.Adapter != "" && !ok {
|
|
return fmt.Errorf("harness output adapter %q is unknown", h.Adapter)
|
|
}
|
|
if req.Gate.MaxConsecutiveFailures > 1 && len(h.Resume) == 0 && len(h.Continue) == 0 {
|
|
return errors.New("stage retries its gate but the harness declares no resume or continue verb")
|
|
}
|
|
if len(h.Resume) > 0 && !slices.ContainsFunc(h.Resume, func(s string) bool { return strings.Contains(s, sessionPlaceholder) }) {
|
|
return fmt.Errorf("harness resume verb must carry %s", sessionPlaceholder)
|
|
}
|
|
if (len(req.CaptureTools) > 0 || h.CentralTools) && !slices.ContainsFunc(h.MCPConfig, func(s string) bool { return strings.Contains(s, mcpConfigPlaceholder) }) {
|
|
return errors.New("stage needs Archie tools but the harness registers no MCP server; its results would be lost")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// harnessArgv is the launch command, then the verb, then the prompt verb
|
|
// when the verb is a resume or continue, with placeholders substituted as
|
|
// raw values rather than shell text.
|
|
func harnessArgv(h *agentrun.HarnessSpec, verb []string, prompt, session, mcpConfig string) []string {
|
|
argv := slices.Clone(h.Launch)
|
|
tails := [][]string{verb}
|
|
if !slices.Equal(verb, h.Prompt) {
|
|
tails = append(tails, h.Prompt)
|
|
}
|
|
if mcpConfig != "" {
|
|
tails = append([][]string{h.MCPConfig}, tails...)
|
|
}
|
|
for _, tail := range tails {
|
|
for _, arg := range tail {
|
|
arg = strings.ReplaceAll(arg, promptPlaceholder, prompt)
|
|
arg = strings.ReplaceAll(arg, sessionPlaceholder, session)
|
|
arg = strings.ReplaceAll(arg, mcpConfigPlaceholder, mcpConfig)
|
|
argv = append(argv, arg)
|
|
}
|
|
}
|
|
return argv
|
|
}
|
|
|
|
// resumeVerb continues the session the last invocation reported, falling
|
|
// back to the most recent session when the CLI reported none. It returns nil
|
|
// when neither is possible: resuming with an empty session ID would start a
|
|
// broken command.
|
|
func resumeVerb(h *agentrun.HarnessSpec, sessionID string) []string {
|
|
if len(h.Resume) > 0 && sessionID != "" {
|
|
return slices.Clone(h.Resume)
|
|
}
|
|
if len(h.Continue) > 0 {
|
|
return slices.Clone(h.Continue)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// invoke runs one harness invocation in its own process group, feeding
|
|
// stdout to out line by line. Cancelling ctx kills the whole group, so a
|
|
// CLI's children cannot outlive the step.
|
|
func invoke(ctx context.Context, workspace string, h *agentrun.HarnessSpec, argv []string, out HarnessOutput, report agentrun.Reporter) error {
|
|
cmd := exec.CommandContext(ctx, argv[0], argv[1:]...)
|
|
cmd.Dir = workspace
|
|
cmd.WaitDelay = killGrace
|
|
// A nil Env would inherit the worker's environment and its credentials.
|
|
cmd.Env = append([]string{}, h.Env...)
|
|
setProcessGroup(cmd)
|
|
if err := runAsHarnessUser(cmd, h.User); err != nil {
|
|
return err
|
|
}
|
|
stdout, err := cmd.StdoutPipe()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var stderr tailBuffer
|
|
cmd.Stderr = io.MultiWriter(&stderr, os.Stderr)
|
|
if err := cmd.Start(); err != nil {
|
|
return fmt.Errorf("start harness: %w", err)
|
|
}
|
|
// Keep the raw harness activity visible in the container's live output.
|
|
output := io.TeeReader(stdout, os.Stdout)
|
|
scanner := bufio.NewScanner(output)
|
|
scanner.Buffer(make([]byte, 64*1024), 16*1024*1024)
|
|
for scanner.Scan() {
|
|
out.Line(scanner.Bytes(), report)
|
|
}
|
|
if err := scanner.Err(); err != nil {
|
|
// Stop the CLI rather than leave it writing into a pipe nobody parses.
|
|
_ = cmd.Cancel()
|
|
_, _ = io.Copy(io.Discard, output)
|
|
_ = cmd.Wait()
|
|
return fmt.Errorf("read harness output: %w", err)
|
|
}
|
|
waitErr := cmd.Wait()
|
|
reapGroup(cmd)
|
|
if err := waitErr; err != nil {
|
|
if ctx.Err() != nil {
|
|
return ctx.Err()
|
|
}
|
|
return fmt.Errorf("harness exited: %w: %s", err, strings.TrimSpace(stderr.String()))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func harnessPrompt(workspace string, req agentrun.Request) string {
|
|
var b strings.Builder
|
|
b.WriteString(req.Mission)
|
|
b.WriteString("\n\n## Rules\n\n")
|
|
b.WriteString(ProjectScopedRules(workspace, req.ExtraRules))
|
|
b.WriteString("\nDo not run git commands that change history, refs or configuration. Archie commits your changes.")
|
|
if req.ReadOnly {
|
|
b.WriteString("\nThis stage is read-only. Do not create, modify or delete any file.")
|
|
}
|
|
if patterns := slices.Concat(req.Protection.Suffixes, req.Protection.Globs); len(patterns) > 0 {
|
|
b.WriteString("\nDo not modify files matching: " + strings.Join(patterns, ", "))
|
|
}
|
|
if len(req.Gate.Commands) > 0 {
|
|
b.WriteString("\nYour work must pass these commands:")
|
|
for _, c := range req.Gate.Commands {
|
|
b.WriteString("\n- " + strings.Join(c.Argv, " "))
|
|
}
|
|
}
|
|
if strings.TrimSpace(req.Notes) != "" {
|
|
b.WriteString("\n\n## Notes\n\n" + req.Notes)
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
// runHarnessGate runs the gate commands confined like the harness: they
|
|
// execute code the harness just wrote, so they get its user, its environment
|
|
// and its process group rather than the worker's.
|
|
func runHarnessGate(ctx context.Context, h *agentrun.HarnessSpec, commands []agentrun.Command, workspace string) (string, error) {
|
|
for _, c := range commands {
|
|
if len(c.Argv) == 0 {
|
|
continue
|
|
}
|
|
cmd := exec.CommandContext(ctx, c.Argv[0], c.Argv[1:]...)
|
|
cmd.Dir = workspace
|
|
cmd.WaitDelay = killGrace
|
|
cmd.Env = append([]string{}, h.Env...)
|
|
setProcessGroup(cmd)
|
|
if err := runAsHarnessUser(cmd, h.User); err != nil {
|
|
return "", err
|
|
}
|
|
output, err := cmd.CombinedOutput()
|
|
reapGroup(cmd)
|
|
switch {
|
|
case c.ExpectFailure && err == nil:
|
|
return fmt.Sprintf("[%s] expected this command to fail, but it passed:\n%s", c.Name, clipGate(output)),
|
|
fmt.Errorf("gate %s: expected failure", c.Name)
|
|
case !c.ExpectFailure && err != nil:
|
|
return fmt.Sprintf("[%s] %s\n%s", c.Name, strings.Join(c.Argv, " "), clipGate(output)), fmt.Errorf("gate %s: %w", c.Name, err)
|
|
}
|
|
}
|
|
return "", nil
|
|
}
|
|
|
|
func clipGate(out []byte) string {
|
|
const limit = 4000
|
|
s := strings.TrimSpace(string(out))
|
|
if len(s) <= limit {
|
|
return s
|
|
}
|
|
return s[:limit/2] + "\n...[truncated]...\n" + s[len(s)-limit/2:]
|
|
}
|
|
|
|
func park(res agentrun.Result, stop, detail string) agentrun.Result {
|
|
res.Status, res.StopReason, res.Detail = StatusParked, stop, detail
|
|
return res
|
|
}
|
|
|
|
func addUsage(a, b agentrun.Usage) agentrun.Usage {
|
|
return agentrun.Usage{
|
|
PromptTokens: a.PromptTokens + b.PromptTokens,
|
|
CompletionTokens: a.CompletionTokens + b.CompletionTokens,
|
|
TotalTokens: a.TotalTokens + b.TotalTokens,
|
|
CachedTokens: a.CachedTokens + b.CachedTokens,
|
|
CacheCreationTokens: a.CacheCreationTokens + b.CacheCreationTokens,
|
|
}
|
|
}
|
|
|
|
// repoState is what the runner compares across an invocation: every ref
|
|
// (HEAD included), the repository configuration, and a content hash of each
|
|
// path git reports as not clean.
|
|
type repoState struct {
|
|
refs map[string]string
|
|
config string
|
|
dirty map[string]string
|
|
}
|
|
|
|
func snapshotRepo(workspace string) (repoState, error) {
|
|
repo, err := git.PlainOpen(workspace)
|
|
if err != nil {
|
|
return repoState{}, err
|
|
}
|
|
state := repoState{refs: map[string]string{}, dirty: map[string]string{}}
|
|
if head, err := repo.Storer.Reference(plumbing.HEAD); err == nil {
|
|
state.refs[plumbing.HEAD.String()] = head.String()
|
|
}
|
|
iter, err := repo.References()
|
|
if err != nil {
|
|
return repoState{}, err
|
|
}
|
|
if err := iter.ForEach(func(ref *plumbing.Reference) error {
|
|
state.refs[ref.Name().String()] = ref.String()
|
|
return nil
|
|
}); err != nil {
|
|
return repoState{}, err
|
|
}
|
|
cfg, err := repo.Config()
|
|
if err != nil {
|
|
return repoState{}, err
|
|
}
|
|
raw, err := cfg.Marshal()
|
|
if err != nil {
|
|
return repoState{}, err
|
|
}
|
|
state.config = string(raw)
|
|
wt, err := repo.Worktree()
|
|
if err != nil {
|
|
return repoState{}, err
|
|
}
|
|
status, err := wt.Status()
|
|
if err != nil {
|
|
return repoState{}, err
|
|
}
|
|
for path, st := range status {
|
|
if st.Worktree == git.Unmodified && st.Staging == git.Unmodified {
|
|
continue
|
|
}
|
|
state.dirty[path] = contentHash(filepath.Join(workspace, path))
|
|
}
|
|
return state, nil
|
|
}
|
|
|
|
func contentHash(path string) string {
|
|
data, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return "absent"
|
|
}
|
|
sum := sha256.Sum256(data)
|
|
return hex.EncodeToString(sum[:])
|
|
}
|
|
|
|
// changedPaths is every path whose state differs across the invocation. A
|
|
// path already dirty and left untouched is not a change of this run.
|
|
func changedPaths(before, after repoState) []string {
|
|
var changed []string
|
|
for path := range mergeKeys(before.dirty, after.dirty) {
|
|
if before.dirty[path] != after.dirty[path] {
|
|
changed = append(changed, path)
|
|
}
|
|
}
|
|
slices.Sort(changed)
|
|
return changed
|
|
}
|
|
|
|
func mergeKeys(a, b map[string]string) map[string]struct{} {
|
|
keys := make(map[string]struct{}, len(a)+len(b))
|
|
for k := range maps.Keys(a) {
|
|
keys[k] = struct{}{}
|
|
}
|
|
for k := range maps.Keys(b) {
|
|
keys[k] = struct{}{}
|
|
}
|
|
return keys
|
|
}
|
|
|
|
func policyViolation(before, after repoState, changes []string, req agentrun.Request) string {
|
|
if !maps.Equal(before.refs, after.refs) {
|
|
var moved []string
|
|
for name := range mergeKeys(before.refs, after.refs) {
|
|
if before.refs[name] != after.refs[name] {
|
|
moved = append(moved, name)
|
|
}
|
|
}
|
|
slices.Sort(moved)
|
|
return "the harness changed git refs: " + strings.Join(moved, ", ")
|
|
}
|
|
if before.config != after.config {
|
|
return "the harness changed the git configuration"
|
|
}
|
|
if req.ReadOnly && len(changes) > 0 {
|
|
return "the harness wrote in a read-only stage: " + strings.Join(changes, ", ")
|
|
}
|
|
if protected := ProtectionMatcher(req.Protection, false); protected != nil {
|
|
var hit []string
|
|
for _, path := range changes {
|
|
if protected(path) {
|
|
hit = append(hit, path)
|
|
}
|
|
}
|
|
if len(hit) > 0 {
|
|
return "the harness modified protected paths: " + strings.Join(hit, ", ")
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
type silentOutput struct{}
|
|
|
|
func (silentOutput) Line([]byte, agentrun.Reporter) {}
|
|
func (silentOutput) SessionID() string { return "" }
|
|
func (silentOutput) Usage() agentrun.Usage { return agentrun.Usage{} }
|
|
|
|
// tailBuffer keeps the last stderrTailBytes written to it.
|
|
type tailBuffer struct{ buf bytes.Buffer }
|
|
|
|
func (t *tailBuffer) Write(p []byte) (int, error) {
|
|
t.buf.Write(p)
|
|
if extra := t.buf.Len() - stderrTailBytes; extra > 0 {
|
|
t.buf.Next(extra)
|
|
}
|
|
return len(p), nil
|
|
}
|
|
|
|
func (t *tailBuffer) String() string { return t.buf.String() }
|
|
|
|
// prepareCaptures writes the capture tools' spec and an MCP config pointing
|
|
// the CLI at archie's MCP server, and creates the captures file the server
|
|
// appends to. The directory belongs to the harness user, whose MCP server
|
|
// writes into it.
|
|
func (r *HarnessRunner) prepareCaptures(req agentrun.Request) (config, captures string, cleanup func(), err error) {
|
|
cleanup = func() {}
|
|
if len(req.CaptureTools) == 0 && !req.Harness.CentralTools {
|
|
return "", "", cleanup, nil
|
|
}
|
|
dir, err := os.MkdirTemp("", "archie-captures-")
|
|
if err != nil {
|
|
return "", "", cleanup, err
|
|
}
|
|
cleanup = func() { _ = os.RemoveAll(dir) }
|
|
fail := func(err error) (string, string, func(), error) {
|
|
cleanup()
|
|
return "", "", func() {}, err
|
|
}
|
|
spec, err := json.Marshal(harnessMCPSpec{Captures: req.CaptureTools, CentralTools: req.Harness.CentralTools})
|
|
if err != nil {
|
|
return fail(err)
|
|
}
|
|
specPath, capturesPath, configPath := filepath.Join(dir, "tools.json"), filepath.Join(dir, "captures.jsonl"), filepath.Join(dir, "mcp.json")
|
|
self, err := os.Executable()
|
|
if err != nil {
|
|
return fail(err)
|
|
}
|
|
server := map[string]any{
|
|
"env": harnessMCPEnvironment(req.Harness.Env),
|
|
"command": self,
|
|
"args": []string{"mcp", "-spec", specPath, "-captures", capturesPath},
|
|
}
|
|
cfg, err := json.Marshal(map[string]any{"mcpServers": map[string]any{"archie": server}})
|
|
if err != nil {
|
|
return fail(err)
|
|
}
|
|
for path, content := range map[string][]byte{specPath: spec, configPath: cfg, capturesPath: nil} {
|
|
if err := os.WriteFile(path, content, 0o600); err != nil {
|
|
return fail(err)
|
|
}
|
|
}
|
|
if err := ownForHarness(req.Harness.User, dir, specPath, configPath, capturesPath); err != nil {
|
|
return fail(err)
|
|
}
|
|
registration, err := harnessMCPRegistration(req.Harness.Adapter, dir, configPath, server)
|
|
if err != nil {
|
|
return fail(err)
|
|
}
|
|
return registration, capturesPath, cleanup, nil
|
|
}
|