package agentexec import ( "bufio" "bytes" "context" "os" "os/exec" "path/filepath" "slices" "strconv" "strings" "sync" "syscall" "testing" "time" ) // fakeHarness is a stand-in coding-agent CLI. FAKE_SCENARIO picks what it // does to the workspace; every invocation's arguments are appended to // FAKE_LOG, and it prints a session line and a tool line for the adapter. const fakeHarness = `#!/bin/sh printf '%s' "$*" | tr '\n' ' ' >> "$FAKE_LOG" echo >> "$FAKE_LOG" case "$FAKE_SCENARIO" in edit) echo hello > notes.txt ;; commit) echo x > a.txt; git add a.txt; git -c user.email=a@b.c -c user.name=a commit -qm sneaky ;; config) git config remote.origin.url https://attacker.invalid/repo.git ;; protected) echo broken >> widget_test.go ;; sleep) sleep 30 & echo $! > "$FAKE_LOG.child"; wait ;; env) env > "$FAKE_LOG.env" ;; capture) cfg=""; prev="" for a in "$@"; do [ "$prev" = "--mcp-config" ] && cfg="$a"; prev="$a"; done out=$(sed -n 's/.*"-captures","\([^"]*\)".*/\1/p' "$cfg") echo '{"tool":"record_finding","args":{"title":"found it"}}' >> "$out" echo '{"tool":"record_finding","args":{"severity":"forged, no title"}}' >> "$out" ;; claude) echo '{"type":"system","subtype":"init","session_id":"C42"}' ;; fix-on-resume) if [ -f .attempted ]; then echo ok > fixed.txt; else touch .attempted; fi ;; esac echo "session=S123" echo "tool=edit_file" ` // lineOutput is a test adapter over fakeHarness's output lines. type lineOutput struct { mu sync.Mutex session string } func (o *lineOutput) Line(line []byte, report ToolCallReporter) { text := string(line) switch { case strings.HasPrefix(text, "session="): o.mu.Lock() o.session = strings.TrimPrefix(text, "session=") o.mu.Unlock() case strings.HasPrefix(text, "tool=") && report != nil: report(ToolCallReport{Tool: strings.TrimPrefix(text, "tool=")}) } } func (o *lineOutput) SessionID() string { o.mu.Lock() defer o.mu.Unlock() return o.session } func (o *lineOutput) Usage() Usage { return Usage{} } func runGit(t *testing.T, dir string, args ...string) { t.Helper() cmd := exec.CommandContext(t.Context(), "git", append([]string{"-c", "user.email=a@b.c", "-c", "user.name=a"}, args...)...) cmd.Dir = dir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git %v: %v\n%s", args, err, out) } } type fixture struct { workspace string log string req Request } func newFixture(t *testing.T, scenario string) *fixture { t.Helper() ws := t.TempDir() runGit(t, ws, "init", "-q") if err := os.WriteFile(filepath.Join(ws, "widget_test.go"), []byte("package widget\n"), 0o600); err != nil { t.Fatal(err) } runGit(t, ws, "add", ".") runGit(t, ws, "commit", "-qm", "base") runGit(t, ws, "remote", "add", "origin", "https://forge.invalid/repo.git") bin := filepath.Join(t.TempDir(), "fake-harness") if err := os.WriteFile(bin, []byte(fakeHarness), 0o700); err != nil { t.Fatal(err) } log := filepath.Join(t.TempDir(), "invocations") return &fixture{workspace: ws, log: log, req: Request{ Version: ProtocolVersion, TaskID: 1, Attempt: 1, Stage: "implement", Model: "harness", Mission: "Write the notes file.", Budget: Budget{WallClock: 20 * time.Second}, Harness: &HarnessSpec{ Env: []string{"PATH=" + os.Getenv("PATH"), "FAKE_SCENARIO=" + scenario, "FAKE_LOG=" + log}, Launch: []string{"/bin/sh", bin}, Prompt: []string{"-p", "{{.Prompt}}"}, Resume: []string{"--resume", "{{.SessionID}}"}, }, }} } func (f *fixture) run(t *testing.T, ctx context.Context) (Result, []ToolCallReport) { t.Helper() var mu sync.Mutex var calls []ToolCallReport res, err := NewHarnessRunner(func() HarnessOutput { return &lineOutput{} }).Run(ctx, f.workspace, f.req, func(r ToolCallReport) { mu.Lock() calls = append(calls, r) mu.Unlock() }) if err != nil { t.Fatalf("Run: %v", err) } return res, calls } func (f *fixture) invocations(t *testing.T) []string { t.Helper() raw, err := os.ReadFile(f.log) if err != nil { return nil } var out []string sc := bufio.NewScanner(bytes.NewReader(raw)) for sc.Scan() { out = append(out, sc.Text()) } return out } func TestHarnessRunsTheMissionAndReportsChanges(t *testing.T) { f := newFixture(t, "edit") res, calls := f.run(t, t.Context()) if res.Status != StatusPassed { t.Fatalf("status %q (%s), want passed", res.Status, res.StopReason) } if !slices.Equal(res.Changes, []string{"notes.txt"}) { t.Fatalf("changes %v, want [notes.txt]", res.Changes) } if len(calls) != 1 || calls[0].Tool != "edit_file" { t.Fatalf("tool calls %v, want one edit_file", calls) } inv := f.invocations(t) if len(inv) != 1 || !strings.HasPrefix(inv[0], "-p ") || !strings.Contains(inv[0], "Write the notes file.") { t.Fatalf("invocations %q, want one headless prompt carrying the mission", inv) } } func TestHarnessGuaranteesHoldWhateverTheHarnessDoes(t *testing.T) { tests := []struct { name string scenario string setup func(*Request) want string }{ {"a commit fails the step", "commit", nil, "git"}, {"rewriting the git config fails the step", "config", nil, "git"}, {"editing a protected path fails the step", "protected", func(r *Request) { r.Protection = Protection{Suffixes: []string{"_test.go"}} }, "widget_test.go"}, {"writing in a read-only stage fails the step", "edit", func(r *Request) { r.ReadOnly = true }, "notes.txt"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { f := newFixture(t, tt.scenario) if tt.setup != nil { tt.setup(&f.req) } res, _ := f.run(t, t.Context()) if res.Status != StatusParked || !strings.Contains(res.StopReason, StopPolicy) || !strings.Contains(res.Detail, tt.want) { t.Fatalf("status %q stop %q detail %q; want parked on policy naming %q", res.Status, res.StopReason, res.Detail, tt.want) } }) } } func TestHarnessIgnoresChangesThatPredateTheRun(t *testing.T) { f := newFixture(t, "edit") if err := os.WriteFile(filepath.Join(f.workspace, "earlier.txt"), []byte("from a previous stage\n"), 0o600); err != nil { t.Fatal(err) } f.req.ReadOnly = false res, _ := f.run(t, t.Context()) if !slices.Equal(res.Changes, []string{"notes.txt"}) { t.Fatalf("changes %v, want only what this run changed", res.Changes) } } func TestHarnessBudgetAndCancellation(t *testing.T) { t.Run("the wall clock kills the harness", func(t *testing.T) { f := newFixture(t, "sleep") f.req.Budget.WallClock = 500 * time.Millisecond start := time.Now() res, _ := f.run(t, t.Context()) if time.Since(start) > 10*time.Second { t.Fatalf("run took %s; the harness outlived its budget", time.Since(start)) } if res.Status != StatusParked || res.StopReason != StopTimedOut { t.Fatalf("status %q stop %q, want parked timed_out", res.Status, res.StopReason) } assertChildDead(t, f.log+".child") }) t.Run("cancelling the run kills the harness", func(t *testing.T) { f := newFixture(t, "sleep") ctx, cancel := context.WithCancel(t.Context()) time.AfterFunc(300*time.Millisecond, cancel) start := time.Now() _, err := NewHarnessRunner(func() HarnessOutput { return &lineOutput{} }).Run(ctx, f.workspace, f.req, nil) if time.Since(start) > 10*time.Second { t.Fatalf("cancel took %s", time.Since(start)) } if err == nil || ctx.Err() == nil { t.Fatalf("Run after cancel returned %v, want the cancellation", err) } assertChildDead(t, f.log+".child") }) } func TestHarnessGateResumesTheSession(t *testing.T) { gate := Gate{Commands: []Command{{Name: "fixed", Argv: []string{"test", "-f", "fixed.txt"}}}, MaxConsecutiveFailures: 3} t.Run("a failed gate resumes the same session with its output", func(t *testing.T) { f := newFixture(t, "fix-on-resume") f.req.Gate = gate res, _ := f.run(t, t.Context()) if res.Status != StatusPassed || res.Iterations != 2 { t.Fatalf("status %q after %d iterations (%s), want passed after 2", res.Status, res.Iterations, res.StopReason) } inv := f.invocations(t) if len(inv) != 2 || !strings.HasPrefix(inv[1], "--resume S123 -p ") || !strings.Contains(inv[1], "[fixed]") { t.Fatalf("invocations %q, want a resume of S123 carrying the gate failure", inv) } }) t.Run("a gate that never passes parks after the cap", func(t *testing.T) { f := newFixture(t, "edit") f.req.Gate = gate res, _ := f.run(t, t.Context()) if res.Status != StatusParked || res.Iterations != 3 { t.Fatalf("status %q after %d iterations, want parked after 3", res.Status, res.Iterations) } }) t.Run("a gate retry without a resume verb is refused before any invocation", func(t *testing.T) { f := newFixture(t, "fix-on-resume") f.req.Gate = gate f.req.Harness.Resume = nil if _, err := NewHarnessRunner(nil).Run(t.Context(), f.workspace, f.req, nil); err == nil || len(f.invocations(t)) != 0 { t.Fatalf("err %v after %d invocations, want a refusal before the harness ran", err, len(f.invocations(t))) } }) t.Run("a single-attempt gate needs no resume verb", func(t *testing.T) { f := newFixture(t, "edit") f.req.Gate = Gate{Commands: gate.Commands, MaxConsecutiveFailures: 1} f.req.Harness.Resume = nil res, _ := f.run(t, t.Context()) if res.Status != StatusParked || res.Iterations != 1 { t.Fatalf("status %q after %d iterations, want parked after 1", res.Status, res.Iterations) } }) } func TestHarnessRequestValidation(t *testing.T) { f := newFixture(t, "edit") f.req.Harness.Prompt = []string{"-p"} if _, err := NewHarnessRunner(nil).Run(t.Context(), f.workspace, f.req, nil); err == nil { t.Fatal("a prompt verb with no {{.Prompt}} placeholder ran; the mission would be silently dropped") } f.req.Harness.Prompt = []string{"-p", "{{.Prompt}}"} f.req.Harness.Adapter = "no-such-cli" if _, err := NewHarnessRunner(nil).Run(t.Context(), f.workspace, f.req, nil); err == nil { t.Fatal("an unknown output adapter ran; its output would be read by nothing") } f.req.Harness = nil if _, err := NewHarnessRunner(nil).Run(t.Context(), f.workspace, f.req, nil); err == nil { t.Fatal("a request without a harness spec ran on the harness runner") } } func TestHarnessReadsOutputWithTheNamedAdapter(t *testing.T) { f := newFixture(t, "claude") f.req.Harness.Adapter = AdapterClaudeCode f.req.Gate = Gate{Commands: []Command{{Name: "never", Argv: []string{"false"}}}, MaxConsecutiveFailures: 2} if _, err := NewHarnessRunner(nil).Run(t.Context(), f.workspace, f.req, nil); err != nil { t.Fatal(err) } inv := f.invocations(t) if len(inv) != 2 || !strings.HasPrefix(inv[1], "--resume C42 ") { t.Fatalf("invocations %q, want a resume of the session the Claude Code adapter read", inv) } } func TestClaudeCodeAdapterRegistersTheMCPServer(t *testing.T) { a, ok := LookupHarnessAdapter(AdapterClaudeCode) if !ok || !slices.ContainsFunc(a.MCPConfig, func(s string) bool { return strings.Contains(s, mcpConfigPlaceholder) }) { t.Fatalf("adapter %+v: capture tools could never reach a Claude Code stage", a) } } func TestHarnessStagesRunEveryStageOnTheTasksHarness(t *testing.T) { inner := &namedRunner{} r := HarnessStages{Runner: inner, Spec: HarnessSpec{Adapter: AdapterClaudeCode}} if _, err := r.Run(t.Context(), "", Request{Stage: "review"}, nil); err != nil { t.Fatal(err) } if inner.got.Harness == nil || inner.got.Harness.Adapter != AdapterClaudeCode { t.Fatalf("stage ran with harness %+v, want the task's", inner.got.Harness) } } type namedRunner struct{ got Request } func (r *namedRunner) Run(_ context.Context, _ string, req Request, _ ToolCallReporter) (Result, error) { r.got = req return Result{}, nil } // assertChildDead checks the harness's own child process did not outlive // it: killing only the CLI would leave its subprocesses running. func assertChildDead(t *testing.T, pidFile string) { t.Helper() raw, err := os.ReadFile(pidFile) if err != nil { t.Fatalf("the harness recorded no child: %v", err) } pid, err := strconv.Atoi(strings.TrimSpace(string(raw))) if err != nil { t.Fatal(err) } deadline := time.Now().Add(5 * time.Second) for syscall.Kill(pid, 0) == nil { if time.Now().After(deadline) { _ = syscall.Kill(pid, syscall.SIGKILL) t.Fatalf("harness child %d outlived the step", pid) } time.Sleep(50 * time.Millisecond) } } func TestHarnessNeverInheritsTheWorkerEnvironment(t *testing.T) { f := newFixture(t, "env") // The fake's own variables are in the worker's environment too, so an // inherited environment would run the fake fully and expose the leak. t.Setenv("FAKE_SCENARIO", "env") t.Setenv("FAKE_LOG", f.log) t.Setenv("STATE_STORE_TOKEN", "worker-secret") t.Setenv("NATS_TOKEN", "worker-secret") f.req.Harness.Env = []string{ "PATH=" + os.Getenv("PATH"), "FAKE_SCENARIO=env", "FAKE_LOG=" + f.log, "HTTPS_PROXY=http://archie:token@archie-egress:3128", } res, _ := f.run(t, t.Context()) if res.Status != StatusPassed { t.Fatalf("status %q (%s)", res.Status, res.Detail) } raw, err := os.ReadFile(f.log + ".env") if err != nil { t.Fatal(err) } env := string(raw) if strings.Contains(env, "worker-secret") { t.Fatalf("the harness saw the worker's credentials:\n%s", env) } if !strings.Contains(env, "HTTPS_PROXY=http://archie:token@archie-egress:3128") { t.Fatalf("the harness did not get its declared environment:\n%s", env) } } func TestHarnessUser(t *testing.T) { tests := []struct { user string uid uint32 gid uint32 wantErr bool }{ {"1000", 1000, 1000, false}, {"1000:1001", 1000, 1001, false}, {"0", 0, 0, true}, {"root", 0, 0, true}, {"not-a-user-anywhere", 0, 0, true}, {"2147483648", 0, 0, true}, {"1000:2147483648", 0, 0, true}, } for _, tt := range tests { cred, err := harnessCredential(tt.user) if tt.wantErr { if err == nil { t.Errorf("harnessCredential(%q) = %+v, want an error", tt.user, cred) } continue } if err != nil || cred.Uid != tt.uid || cred.Gid != tt.gid { t.Errorf("harnessCredential(%q) = %+v, %v; want %d:%d", tt.user, cred, err, tt.uid, tt.gid) } } }