Files
archie/internal/agentexec/harness_runner_test.go
T
sam 21e9e4a3b7 feat(agentexec): run harness invocations as the Kit's user with no inherited environment
A Kit task runs worker and harness in one container, separated by user:
the worker keeps its credentials as root, and every harness invocation
runs as the Kit's user with only its declared environment. A nil env
would inherit the worker's, so the runner always passes an explicit one;
root is refused as a harness user, and a root worker refuses to start a
harness with no user. The PRD's container layout now says one container
per run, with the relay forwarding the proxy, NATS and the State Store.
2026-09-26 03:08:34 +10:00

347 lines
11 KiB
Go

package agentexec
import (
"bufio"
"bytes"
"context"
"os"
"os/exec"
"path/filepath"
"slices"
"strconv"
"strings"
"sync"
"syscall"
"testing"
"time"
)
// fakeHarness is a stand-in coding-agent CLI. FAKE_SCENARIO picks what it
// does to the workspace; every invocation's arguments are appended to
// FAKE_LOG, and it prints a session line and a tool line for the adapter.
const fakeHarness = `#!/bin/sh
printf '%s' "$*" | tr '\n' ' ' >> "$FAKE_LOG"
echo >> "$FAKE_LOG"
case "$FAKE_SCENARIO" in
edit) echo hello > notes.txt ;;
commit) echo x > a.txt; git add a.txt; git -c user.email=a@b.c -c user.name=a commit -qm sneaky ;;
config) git config remote.origin.url https://attacker.invalid/repo.git ;;
protected) echo broken >> widget_test.go ;;
sleep) sleep 30 & echo $! > "$FAKE_LOG.child"; wait ;;
env) env > "$FAKE_LOG.env" ;;
fix-on-resume) if [ -f .attempted ]; then echo ok > fixed.txt; else touch .attempted; fi ;;
esac
echo "session=S123"
echo "tool=edit_file"
`
// lineOutput is a test adapter over fakeHarness's output lines.
type lineOutput struct {
mu sync.Mutex
session string
}
func (o *lineOutput) Line(line []byte, report ToolCallReporter) {
text := string(line)
switch {
case strings.HasPrefix(text, "session="):
o.mu.Lock()
o.session = strings.TrimPrefix(text, "session=")
o.mu.Unlock()
case strings.HasPrefix(text, "tool=") && report != nil:
report(ToolCallReport{Tool: strings.TrimPrefix(text, "tool=")})
}
}
func (o *lineOutput) SessionID() string {
o.mu.Lock()
defer o.mu.Unlock()
return o.session
}
func (o *lineOutput) Usage() Usage { return Usage{} }
func runGit(t *testing.T, dir string, args ...string) {
t.Helper()
cmd := exec.CommandContext(t.Context(), "git", append([]string{"-c", "user.email=a@b.c", "-c", "user.name=a"}, args...)...)
cmd.Dir = dir
if out, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("git %v: %v\n%s", args, err, out)
}
}
type fixture struct {
workspace string
log string
req Request
}
func newFixture(t *testing.T, scenario string) *fixture {
t.Helper()
ws := t.TempDir()
runGit(t, ws, "init", "-q")
if err := os.WriteFile(filepath.Join(ws, "widget_test.go"), []byte("package widget\n"), 0o600); err != nil {
t.Fatal(err)
}
runGit(t, ws, "add", ".")
runGit(t, ws, "commit", "-qm", "base")
runGit(t, ws, "remote", "add", "origin", "https://forge.invalid/repo.git")
bin := filepath.Join(t.TempDir(), "fake-harness")
if err := os.WriteFile(bin, []byte(fakeHarness), 0o700); err != nil {
t.Fatal(err)
}
log := filepath.Join(t.TempDir(), "invocations")
return &fixture{workspace: ws, log: log, req: Request{
Version: ProtocolVersion, TaskID: 1, Attempt: 1, Stage: "implement", Model: "harness",
Mission: "Write the notes file.",
Budget: Budget{WallClock: 20 * time.Second},
Harness: &HarnessSpec{
Env: []string{"PATH=" + os.Getenv("PATH"), "FAKE_SCENARIO=" + scenario, "FAKE_LOG=" + log},
Launch: []string{"/bin/sh", bin},
Prompt: []string{"-p", "{{.Prompt}}"},
Resume: []string{"--resume", "{{.SessionID}}"},
},
}}
}
func (f *fixture) run(t *testing.T, ctx context.Context) (Result, []ToolCallReport) {
t.Helper()
var mu sync.Mutex
var calls []ToolCallReport
res, err := NewHarnessRunner(func() HarnessOutput { return &lineOutput{} }).Run(ctx, f.workspace, f.req, func(r ToolCallReport) {
mu.Lock()
calls = append(calls, r)
mu.Unlock()
})
if err != nil {
t.Fatalf("Run: %v", err)
}
return res, calls
}
func (f *fixture) invocations(t *testing.T) []string {
t.Helper()
raw, err := os.ReadFile(f.log)
if err != nil {
return nil
}
var out []string
sc := bufio.NewScanner(bytes.NewReader(raw))
for sc.Scan() {
out = append(out, sc.Text())
}
return out
}
func TestHarnessRunsTheMissionAndReportsChanges(t *testing.T) {
f := newFixture(t, "edit")
res, calls := f.run(t, t.Context())
if res.Status != StatusPassed {
t.Fatalf("status %q (%s), want passed", res.Status, res.StopReason)
}
if !slices.Equal(res.Changes, []string{"notes.txt"}) {
t.Fatalf("changes %v, want [notes.txt]", res.Changes)
}
if len(calls) != 1 || calls[0].Tool != "edit_file" {
t.Fatalf("tool calls %v, want one edit_file", calls)
}
inv := f.invocations(t)
if len(inv) != 1 || !strings.HasPrefix(inv[0], "-p ") || !strings.Contains(inv[0], "Write the notes file.") {
t.Fatalf("invocations %q, want one headless prompt carrying the mission", inv)
}
}
func TestHarnessGuaranteesHoldWhateverTheHarnessDoes(t *testing.T) {
tests := []struct {
name string
scenario string
setup func(*Request)
want string
}{
{"a commit fails the step", "commit", nil, "git"},
{"rewriting the git config fails the step", "config", nil, "git"},
{"editing a protected path fails the step", "protected", func(r *Request) { r.Protection = Protection{Suffixes: []string{"_test.go"}} }, "widget_test.go"},
{"writing in a read-only stage fails the step", "edit", func(r *Request) { r.ReadOnly = true }, "notes.txt"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
f := newFixture(t, tt.scenario)
if tt.setup != nil {
tt.setup(&f.req)
}
res, _ := f.run(t, t.Context())
if res.Status != StatusParked || !strings.Contains(res.StopReason, StopPolicy) || !strings.Contains(res.Detail, tt.want) {
t.Fatalf("status %q stop %q detail %q; want parked on policy naming %q", res.Status, res.StopReason, res.Detail, tt.want)
}
})
}
}
func TestHarnessIgnoresChangesThatPredateTheRun(t *testing.T) {
f := newFixture(t, "edit")
if err := os.WriteFile(filepath.Join(f.workspace, "earlier.txt"), []byte("from a previous stage\n"), 0o600); err != nil {
t.Fatal(err)
}
f.req.ReadOnly = false
res, _ := f.run(t, t.Context())
if !slices.Equal(res.Changes, []string{"notes.txt"}) {
t.Fatalf("changes %v, want only what this run changed", res.Changes)
}
}
func TestHarnessBudgetAndCancellation(t *testing.T) {
t.Run("the wall clock kills the harness", func(t *testing.T) {
f := newFixture(t, "sleep")
f.req.Budget.WallClock = 500 * time.Millisecond
start := time.Now()
res, _ := f.run(t, t.Context())
if time.Since(start) > 10*time.Second {
t.Fatalf("run took %s; the harness outlived its budget", time.Since(start))
}
if res.Status != StatusParked || res.StopReason != StopTimedOut {
t.Fatalf("status %q stop %q, want parked timed_out", res.Status, res.StopReason)
}
assertChildDead(t, f.log+".child")
})
t.Run("cancelling the run kills the harness", func(t *testing.T) {
f := newFixture(t, "sleep")
ctx, cancel := context.WithCancel(t.Context())
time.AfterFunc(300*time.Millisecond, cancel)
start := time.Now()
_, err := NewHarnessRunner(func() HarnessOutput { return &lineOutput{} }).Run(ctx, f.workspace, f.req, nil)
if time.Since(start) > 10*time.Second {
t.Fatalf("cancel took %s", time.Since(start))
}
if err == nil || ctx.Err() == nil {
t.Fatalf("Run after cancel returned %v, want the cancellation", err)
}
assertChildDead(t, f.log+".child")
})
}
func TestHarnessGateResumesTheSession(t *testing.T) {
gate := Gate{Commands: []Command{{Name: "fixed", Argv: []string{"test", "-f", "fixed.txt"}}}, MaxConsecutiveFailures: 3}
t.Run("a failed gate resumes the same session with its output", func(t *testing.T) {
f := newFixture(t, "fix-on-resume")
f.req.Gate = gate
res, _ := f.run(t, t.Context())
if res.Status != StatusPassed || res.Iterations != 2 {
t.Fatalf("status %q after %d iterations (%s), want passed after 2", res.Status, res.Iterations, res.StopReason)
}
inv := f.invocations(t)
if len(inv) != 2 || !strings.HasPrefix(inv[1], "--resume S123 -p ") || !strings.Contains(inv[1], "[fixed]") {
t.Fatalf("invocations %q, want a resume of S123 carrying the gate failure", inv)
}
})
t.Run("a gate that never passes parks after the cap", func(t *testing.T) {
f := newFixture(t, "edit")
f.req.Gate = gate
res, _ := f.run(t, t.Context())
if res.Status != StatusParked || res.Iterations != 3 {
t.Fatalf("status %q after %d iterations, want parked after 3", res.Status, res.Iterations)
}
})
t.Run("without a resume verb there is one iteration", func(t *testing.T) {
f := newFixture(t, "fix-on-resume")
f.req.Gate = gate
f.req.Harness.Resume = nil
res, _ := f.run(t, t.Context())
if res.Status != StatusParked || res.Iterations != 1 || len(f.invocations(t)) != 1 {
t.Fatalf("status %q after %d iterations and %d invocations, want parked after 1", res.Status, res.Iterations, len(f.invocations(t)))
}
})
}
func TestHarnessRequestValidation(t *testing.T) {
f := newFixture(t, "edit")
f.req.Harness.Prompt = []string{"-p"}
if _, err := NewHarnessRunner(nil).Run(t.Context(), f.workspace, f.req, nil); err == nil {
t.Fatal("a prompt verb with no {{.Prompt}} placeholder ran; the mission would be silently dropped")
}
f.req.Harness = nil
if _, err := NewHarnessRunner(nil).Run(t.Context(), f.workspace, f.req, nil); err == nil {
t.Fatal("a request without a harness spec ran on the harness runner")
}
}
// assertChildDead checks the harness's own child process did not outlive
// it: killing only the CLI would leave its subprocesses running.
func assertChildDead(t *testing.T, pidFile string) {
t.Helper()
raw, err := os.ReadFile(pidFile)
if err != nil {
t.Fatalf("the harness recorded no child: %v", err)
}
pid, err := strconv.Atoi(strings.TrimSpace(string(raw)))
if err != nil {
t.Fatal(err)
}
deadline := time.Now().Add(5 * time.Second)
for syscall.Kill(pid, 0) == nil {
if time.Now().After(deadline) {
_ = syscall.Kill(pid, syscall.SIGKILL)
t.Fatalf("harness child %d outlived the step", pid)
}
time.Sleep(50 * time.Millisecond)
}
}
func TestHarnessNeverInheritsTheWorkerEnvironment(t *testing.T) {
f := newFixture(t, "env")
// The fake's own variables are in the worker's environment too, so an
// inherited environment would run the fake fully and expose the leak.
t.Setenv("FAKE_SCENARIO", "env")
t.Setenv("FAKE_LOG", f.log)
t.Setenv("STATE_STORE_TOKEN", "worker-secret")
t.Setenv("NATS_TOKEN", "worker-secret")
f.req.Harness.Env = []string{
"PATH=" + os.Getenv("PATH"),
"FAKE_SCENARIO=env", "FAKE_LOG=" + f.log,
"HTTPS_PROXY=http://archie:token@archie-egress:3128",
}
res, _ := f.run(t, t.Context())
if res.Status != StatusPassed {
t.Fatalf("status %q (%s)", res.Status, res.Detail)
}
raw, err := os.ReadFile(f.log + ".env")
if err != nil {
t.Fatal(err)
}
env := string(raw)
if strings.Contains(env, "worker-secret") {
t.Fatalf("the harness saw the worker's credentials:\n%s", env)
}
if !strings.Contains(env, "HTTPS_PROXY=http://archie:token@archie-egress:3128") {
t.Fatalf("the harness did not get its declared environment:\n%s", env)
}
}
func TestHarnessUser(t *testing.T) {
tests := []struct {
user string
uid uint32
gid uint32
wantErr bool
}{
{"1000", 1000, 1000, false},
{"1000:1001", 1000, 1001, false},
{"0", 0, 0, true},
{"root", 0, 0, true},
{"not-a-user-anywhere", 0, 0, true},
}
for _, tt := range tests {
cred, err := harnessCredential(tt.user)
if tt.wantErr {
if err == nil {
t.Errorf("harnessCredential(%q) = %+v, want an error", tt.user, cred)
}
continue
}
if err != nil || cred.Uid != tt.uid || cred.Gid != tt.gid {
t.Errorf("harnessCredential(%q) = %+v, %v; want %d:%d", tt.user, cred, err, tt.uid, tt.gid)
}
}
}