Files
archie/internal/agentexec/harness_runner_test.go
T
sam 61381450e2 test(agentexec): make seeded git repos independent of host signing config
The temp-repo fixtures in agentexec, domain/workflow and agentworker seeded
a repository and committed without disabling gpg signing, so a host with
commit.gpgsign=true and an agent-only SSH signing key failed those tests at
the fixture's own git call ("Couldn't get agent socket?").

Each producer now writes commit.gpgsign=false into the temporary repository's
own config, which also covers commits made inside the seeded repo by a
subprocess (the fake harness). A regression test per package pins the
fixture's hermeticity under a hostile global git config.
2026-10-02 03:06:48 +10:00

429 lines
15 KiB
Go

package agentexec
import (
"bufio"
"bytes"
"context"
"os"
"os/exec"
"path/filepath"
"slices"
"strconv"
"strings"
"sync"
"syscall"
"testing"
"time"
)
// fakeHarness is a stand-in coding-agent CLI. FAKE_SCENARIO picks what it
// does to the workspace; every invocation's arguments are appended to
// FAKE_LOG, and it prints a session line and a tool line for the adapter.
const fakeHarness = `#!/bin/sh
printf '%s' "$*" | tr '\n' ' ' >> "$FAKE_LOG"
echo >> "$FAKE_LOG"
case "$FAKE_SCENARIO" in
edit) echo hello > notes.txt ;;
commit) echo x > a.txt; git add a.txt; git -c user.email=a@b.c -c user.name=a commit -qm sneaky ;;
config) git config remote.origin.url https://attacker.invalid/repo.git ;;
protected) echo broken >> widget_test.go ;;
sleep) sleep 30 & echo $! > "$FAKE_LOG.child"; wait ;;
env) env > "$FAKE_LOG.env" ;;
capture)
cfg=""; prev=""
for a in "$@"; do [ "$prev" = "--mcp-config" ] && cfg="$a"; prev="$a"; done
out=$(sed -n 's/.*"-captures","\([^"]*\)".*/\1/p' "$cfg")
echo '{"tool":"record_finding","args":{"title":"found it"}}' >> "$out"
echo '{"tool":"record_finding","args":{"severity":"forged, no title"}}' >> "$out" ;;
claude) echo '{"type":"system","subtype":"init","session_id":"C42"}' ;;
fix-on-resume) if [ -f .attempted ]; then echo ok > fixed.txt; else touch .attempted; fi ;;
esac
echo "session=S123"
echo "tool=edit_file"
`
// lineOutput is a test adapter over fakeHarness's output lines.
type lineOutput struct {
mu sync.Mutex
session string
}
func (o *lineOutput) Line(line []byte, report ToolCallReporter) {
text := string(line)
switch {
case strings.HasPrefix(text, "session="):
o.mu.Lock()
o.session = strings.TrimPrefix(text, "session=")
o.mu.Unlock()
case strings.HasPrefix(text, "tool=") && report != nil:
report(ToolCallReport{Tool: strings.TrimPrefix(text, "tool=")})
}
}
func (o *lineOutput) SessionID() string {
o.mu.Lock()
defer o.mu.Unlock()
return o.session
}
func (o *lineOutput) Usage() Usage { return Usage{} }
func runGit(t *testing.T, dir string, args ...string) {
t.Helper()
cmd := exec.CommandContext(t.Context(), "git", append([]string{"-c", "user.email=a@b.c", "-c", "user.name=a"}, args...)...)
cmd.Dir = dir
if out, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("git %v: %v\n%s", args, err, out)
}
}
// hostileSigningConfig writes a global git config that makes every commit
// fail unless the fixture disables signing in the repository's own config.
func hostileSigningConfig(t *testing.T) string {
t.Helper()
path := filepath.Join(t.TempDir(), "hostile.gitconfig")
const cfg = "[user]\n\tname = Hostile\n\temail = hostile@example.com\n\tsigningkey = /nonexistent/signing-key\n[gpg]\n\tformat = ssh\n[commit]\n\tgpgsign = true\n"
if err := os.WriteFile(path, []byte(cfg), 0o600); err != nil {
t.Fatal(err)
}
return path
}
// TestFixtureSeedsARepositoryThatIgnoresHostSigningConfig pins the fixture's
// hermeticity: it must not depend on the developer's global git identity or a
// signing agent.
func TestFixtureSeedsARepositoryThatIgnoresHostSigningConfig(t *testing.T) {
t.Setenv("GIT_CONFIG_GLOBAL", hostileSigningConfig(t))
t.Setenv("GIT_CONFIG_NOSYSTEM", "1")
newFixture(t, "edit")
}
type fixture struct {
workspace string
log string
req Request
}
func newFixture(t *testing.T, scenario string) *fixture {
t.Helper()
ws := t.TempDir()
runGit(t, ws, "init", "-q")
runGit(t, ws, "config", "commit.gpgsign", "false")
if err := os.WriteFile(filepath.Join(ws, "widget_test.go"), []byte("package widget\n"), 0o600); err != nil {
t.Fatal(err)
}
runGit(t, ws, "add", ".")
runGit(t, ws, "commit", "-qm", "base")
runGit(t, ws, "remote", "add", "origin", "https://forge.invalid/repo.git")
bin := filepath.Join(t.TempDir(), "fake-harness")
if err := os.WriteFile(bin, []byte(fakeHarness), 0o700); err != nil {
t.Fatal(err)
}
log := filepath.Join(t.TempDir(), "invocations")
return &fixture{workspace: ws, log: log, req: Request{
Version: ProtocolVersion, TaskID: 1, Attempt: 1, Stage: "implement", Model: "harness",
Mission: "Write the notes file.",
Budget: Budget{WallClock: 20 * time.Second},
Harness: &HarnessSpec{
Env: []string{"PATH=" + os.Getenv("PATH"), "FAKE_SCENARIO=" + scenario, "FAKE_LOG=" + log},
Launch: []string{"/bin/sh", bin},
Prompt: []string{"-p", "{{.Prompt}}"},
Resume: []string{"--resume", "{{.SessionID}}"},
},
}}
}
func (f *fixture) run(t *testing.T, ctx context.Context) (Result, []ToolCallReport) {
t.Helper()
var mu sync.Mutex
var calls []ToolCallReport
res, err := NewHarnessRunner(func() HarnessOutput { return &lineOutput{} }).Run(ctx, f.workspace, f.req, func(r ToolCallReport) {
mu.Lock()
calls = append(calls, r)
mu.Unlock()
})
if err != nil {
t.Fatalf("Run: %v", err)
}
return res, calls
}
func (f *fixture) invocations(t *testing.T) []string {
t.Helper()
raw, err := os.ReadFile(f.log)
if err != nil {
return nil
}
var out []string
sc := bufio.NewScanner(bytes.NewReader(raw))
for sc.Scan() {
out = append(out, sc.Text())
}
return out
}
func TestHarnessRunsTheMissionAndReportsChanges(t *testing.T) {
f := newFixture(t, "edit")
res, calls := f.run(t, t.Context())
if res.Status != StatusPassed {
t.Fatalf("status %q (%s), want passed", res.Status, res.StopReason)
}
if !slices.Equal(res.Changes, []string{"notes.txt"}) {
t.Fatalf("changes %v, want [notes.txt]", res.Changes)
}
if len(calls) != 1 || calls[0].Tool != "edit_file" {
t.Fatalf("tool calls %v, want one edit_file", calls)
}
inv := f.invocations(t)
if len(inv) != 1 || !strings.HasPrefix(inv[0], "-p ") || !strings.Contains(inv[0], "Write the notes file.") {
t.Fatalf("invocations %q, want one headless prompt carrying the mission", inv)
}
}
func TestHarnessGuaranteesHoldWhateverTheHarnessDoes(t *testing.T) {
tests := []struct {
name string
scenario string
setup func(*Request)
want string
}{
{"a commit fails the step", "commit", nil, "git"},
{"rewriting the git config fails the step", "config", nil, "git"},
{"editing a protected path fails the step", "protected", func(r *Request) { r.Protection = Protection{Suffixes: []string{"_test.go"}} }, "widget_test.go"},
{"writing in a read-only stage fails the step", "edit", func(r *Request) { r.ReadOnly = true }, "notes.txt"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
f := newFixture(t, tt.scenario)
if tt.setup != nil {
tt.setup(&f.req)
}
res, _ := f.run(t, t.Context())
if res.Status != StatusParked || !strings.Contains(res.StopReason, StopPolicy) || !strings.Contains(res.Detail, tt.want) {
t.Fatalf("status %q stop %q detail %q; want parked on policy naming %q", res.Status, res.StopReason, res.Detail, tt.want)
}
})
}
}
func TestHarnessIgnoresChangesThatPredateTheRun(t *testing.T) {
f := newFixture(t, "edit")
if err := os.WriteFile(filepath.Join(f.workspace, "earlier.txt"), []byte("from a previous stage\n"), 0o600); err != nil {
t.Fatal(err)
}
f.req.ReadOnly = false
res, _ := f.run(t, t.Context())
if !slices.Equal(res.Changes, []string{"notes.txt"}) {
t.Fatalf("changes %v, want only what this run changed", res.Changes)
}
}
func TestHarnessBudgetAndCancellation(t *testing.T) {
t.Run("the wall clock kills the harness", func(t *testing.T) {
f := newFixture(t, "sleep")
f.req.Budget.WallClock = 500 * time.Millisecond
start := time.Now()
res, _ := f.run(t, t.Context())
if time.Since(start) > 10*time.Second {
t.Fatalf("run took %s; the harness outlived its budget", time.Since(start))
}
if res.Status != StatusParked || res.StopReason != StopTimedOut {
t.Fatalf("status %q stop %q, want parked timed_out", res.Status, res.StopReason)
}
assertChildDead(t, f.log+".child")
})
t.Run("cancelling the run kills the harness", func(t *testing.T) {
f := newFixture(t, "sleep")
ctx, cancel := context.WithCancel(t.Context())
time.AfterFunc(300*time.Millisecond, cancel)
start := time.Now()
_, err := NewHarnessRunner(func() HarnessOutput { return &lineOutput{} }).Run(ctx, f.workspace, f.req, nil)
if time.Since(start) > 10*time.Second {
t.Fatalf("cancel took %s", time.Since(start))
}
if err == nil || ctx.Err() == nil {
t.Fatalf("Run after cancel returned %v, want the cancellation", err)
}
assertChildDead(t, f.log+".child")
})
}
func TestHarnessGateResumesTheSession(t *testing.T) {
gate := Gate{Commands: []Command{{Name: "fixed", Argv: []string{"test", "-f", "fixed.txt"}}}, MaxConsecutiveFailures: 3}
t.Run("a failed gate resumes the same session with its output", func(t *testing.T) {
f := newFixture(t, "fix-on-resume")
f.req.Gate = gate
res, _ := f.run(t, t.Context())
if res.Status != StatusPassed || res.Iterations != 2 {
t.Fatalf("status %q after %d iterations (%s), want passed after 2", res.Status, res.Iterations, res.StopReason)
}
inv := f.invocations(t)
if len(inv) != 2 || !strings.HasPrefix(inv[1], "--resume S123 -p ") || !strings.Contains(inv[1], "[fixed]") {
t.Fatalf("invocations %q, want a resume of S123 carrying the gate failure", inv)
}
})
t.Run("a gate that never passes parks after the cap", func(t *testing.T) {
f := newFixture(t, "edit")
f.req.Gate = gate
res, _ := f.run(t, t.Context())
if res.Status != StatusParked || res.Iterations != 3 {
t.Fatalf("status %q after %d iterations, want parked after 3", res.Status, res.Iterations)
}
})
t.Run("a gate retry without a resume verb is refused before any invocation", func(t *testing.T) {
f := newFixture(t, "fix-on-resume")
f.req.Gate = gate
f.req.Harness.Resume = nil
if _, err := NewHarnessRunner(nil).Run(t.Context(), f.workspace, f.req, nil); err == nil || len(f.invocations(t)) != 0 {
t.Fatalf("err %v after %d invocations, want a refusal before the harness ran", err, len(f.invocations(t)))
}
})
t.Run("a single-attempt gate needs no resume verb", func(t *testing.T) {
f := newFixture(t, "edit")
f.req.Gate = Gate{Commands: gate.Commands, MaxConsecutiveFailures: 1}
f.req.Harness.Resume = nil
res, _ := f.run(t, t.Context())
if res.Status != StatusParked || res.Iterations != 1 {
t.Fatalf("status %q after %d iterations, want parked after 1", res.Status, res.Iterations)
}
})
}
func TestHarnessRequestValidation(t *testing.T) {
f := newFixture(t, "edit")
f.req.Harness.Prompt = []string{"-p"}
if _, err := NewHarnessRunner(nil).Run(t.Context(), f.workspace, f.req, nil); err == nil {
t.Fatal("a prompt verb with no {{.Prompt}} placeholder ran; the mission would be silently dropped")
}
f.req.Harness.Prompt = []string{"-p", "{{.Prompt}}"}
f.req.Harness.Adapter = "no-such-cli"
if _, err := NewHarnessRunner(nil).Run(t.Context(), f.workspace, f.req, nil); err == nil {
t.Fatal("an unknown output adapter ran; its output would be read by nothing")
}
f.req.Harness = nil
if _, err := NewHarnessRunner(nil).Run(t.Context(), f.workspace, f.req, nil); err == nil {
t.Fatal("a request without a harness spec ran on the harness runner")
}
}
func TestHarnessReadsOutputWithTheNamedAdapter(t *testing.T) {
f := newFixture(t, "claude")
f.req.Harness.Adapter = AdapterClaudeCode
f.req.Gate = Gate{Commands: []Command{{Name: "never", Argv: []string{"false"}}}, MaxConsecutiveFailures: 2}
if _, err := NewHarnessRunner(nil).Run(t.Context(), f.workspace, f.req, nil); err != nil {
t.Fatal(err)
}
inv := f.invocations(t)
if len(inv) != 2 || !strings.HasPrefix(inv[1], "--resume C42 ") {
t.Fatalf("invocations %q, want a resume of the session the Claude Code adapter read", inv)
}
}
func TestClaudeCodeAdapterRegistersTheMCPServer(t *testing.T) {
a, ok := LookupHarnessAdapter(AdapterClaudeCode)
if !ok || !slices.ContainsFunc(a.MCPConfig, func(s string) bool { return strings.Contains(s, mcpConfigPlaceholder) }) {
t.Fatalf("adapter %+v: capture tools could never reach a Claude Code stage", a)
}
}
func TestHarnessStagesRunEveryStageOnTheTasksHarness(t *testing.T) {
inner := &namedRunner{}
r := HarnessStages{Runner: inner, Spec: HarnessSpec{Adapter: AdapterClaudeCode}}
if _, err := r.Run(t.Context(), "", Request{Stage: "review"}, nil); err != nil {
t.Fatal(err)
}
if inner.got.Harness == nil || inner.got.Harness.Adapter != AdapterClaudeCode {
t.Fatalf("stage ran with harness %+v, want the task's", inner.got.Harness)
}
}
type namedRunner struct{ got Request }
func (r *namedRunner) Run(_ context.Context, _ string, req Request, _ ToolCallReporter) (Result, error) {
r.got = req
return Result{}, nil
}
// assertChildDead checks the harness's own child process did not outlive
// it: killing only the CLI would leave its subprocesses running.
func assertChildDead(t *testing.T, pidFile string) {
t.Helper()
raw, err := os.ReadFile(pidFile)
if err != nil {
t.Fatalf("the harness recorded no child: %v", err)
}
pid, err := strconv.Atoi(strings.TrimSpace(string(raw)))
if err != nil {
t.Fatal(err)
}
deadline := time.Now().Add(5 * time.Second)
for syscall.Kill(pid, 0) == nil {
if time.Now().After(deadline) {
_ = syscall.Kill(pid, syscall.SIGKILL)
t.Fatalf("harness child %d outlived the step", pid)
}
time.Sleep(50 * time.Millisecond)
}
}
func TestHarnessNeverInheritsTheWorkerEnvironment(t *testing.T) {
f := newFixture(t, "env")
// The fake's own variables are in the worker's environment too, so an
// inherited environment would run the fake fully and expose the leak.
t.Setenv("FAKE_SCENARIO", "env")
t.Setenv("FAKE_LOG", f.log)
t.Setenv("STATE_STORE_TOKEN", "worker-secret")
t.Setenv("NATS_TOKEN", "worker-secret")
f.req.Harness.Env = []string{
"PATH=" + os.Getenv("PATH"),
"FAKE_SCENARIO=env", "FAKE_LOG=" + f.log,
"HTTPS_PROXY=http://archie:token@archie-egress:3128",
}
res, _ := f.run(t, t.Context())
if res.Status != StatusPassed {
t.Fatalf("status %q (%s)", res.Status, res.Detail)
}
raw, err := os.ReadFile(f.log + ".env")
if err != nil {
t.Fatal(err)
}
env := string(raw)
if strings.Contains(env, "worker-secret") {
t.Fatalf("the harness saw the worker's credentials:\n%s", env)
}
if !strings.Contains(env, "HTTPS_PROXY=http://archie:token@archie-egress:3128") {
t.Fatalf("the harness did not get its declared environment:\n%s", env)
}
}
func TestHarnessUser(t *testing.T) {
tests := []struct {
user string
uid uint32
gid uint32
wantErr bool
}{
{"1000", 1000, 1000, false},
{"1000:1001", 1000, 1001, false},
{"0", 0, 0, true},
{"root", 0, 0, true},
{"not-a-user-anywhere", 0, 0, true},
{"2147483648", 0, 0, true},
{"1000:2147483648", 0, 0, true},
}
for _, tt := range tests {
cred, err := harnessCredential(tt.user)
if tt.wantErr {
if err == nil {
t.Errorf("harnessCredential(%q) = %+v, want an error", tt.user, cred)
}
continue
}
if err != nil || cred.Uid != tt.uid || cred.Gid != tt.gid {
t.Errorf("harnessCredential(%q) = %+v, %v; want %d:%d", tt.user, cred, err, tt.uid, tt.gid)
}
}
}