feat(agentexec): harness runner driving a Kit's agent headlessly
A stage whose request carries a HarnessSpec runs the Kit's CLI as a subprocess in its own process group, with the mission and archie's rules as the prompt. After every invocation the runner compares HEAD, every ref and the git configuration, and the worktree against a pre-run baseline: any ref or config change, a protected-path edit, or a write in a read-only stage parks the step whatever the CLI claimed. The runner runs the gate itself and resumes the same session with the gate's output, up to the gate's failure cap. The wall clock and cancellation kill the whole process group, so the CLI's children cannot outlive the step.
This commit is contained in:
5 files changed
+758
No files matched your search
@@ -0,0 +1,9 @@
|
||||
//go:build !unix
|
||||
|
||||
package agentexec
|
||||
|
||||
import "os/exec"
|
||||
|
||||
// setProcessGroup falls back to killing the harness process alone where
|
||||
// process groups are unavailable.
|
||||
func setProcessGroup(*exec.Cmd) {}
|
||||
@@ -0,0 +1,15 @@
|
||||
//go:build unix
|
||||
|
||||
package agentexec
|
||||
|
||||
import (
|
||||
"os/exec"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// setProcessGroup puts the harness in its own process group and makes
|
||||
// cancellation kill the whole group.
|
||||
func setProcessGroup(cmd *exec.Cmd) {
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||
cmd.Cancel = func() error { return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) }
|
||||
}
|
||||
@@ -0,0 +1,431 @@
|
||||
package agentexec
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
git "github.com/go-git/go-git/v6"
|
||||
"github.com/go-git/go-git/v6/plumbing"
|
||||
)
|
||||
|
||||
// Harness outcomes beyond StatusPassed. The strings match the built-in
|
||||
// loop's, so a stage reads the same whichever runner ran it.
|
||||
const (
|
||||
StatusParked = "parked"
|
||||
StopTimedOut = "timed_out"
|
||||
StopPolicy = "policy_violation"
|
||||
StopGate = "gate_failed"
|
||||
StopExited = "harness_exited"
|
||||
)
|
||||
|
||||
const (
|
||||
promptPlaceholder = "{{.Prompt}}"
|
||||
sessionPlaceholder = "{{.SessionID}}"
|
||||
stderrTailBytes = 2000
|
||||
killGrace = 2 * time.Second
|
||||
)
|
||||
|
||||
// HarnessOutput reads one harness invocation's standard output. An adapter
|
||||
// knows its CLI's stream format; it reports tool calls as they complete and
|
||||
// the session to resume.
|
||||
type HarnessOutput interface {
|
||||
Line(line []byte, report ToolCallReporter)
|
||||
SessionID() string
|
||||
Usage() Usage
|
||||
}
|
||||
|
||||
// HarnessRunner runs a stage on an external coding-agent CLI. Archie's
|
||||
// guarantees do not depend on the CLI obeying its prompt: the runner checks
|
||||
// the worktree and the repository's refs and configuration after every
|
||||
// invocation, runs the gate itself, and owns the process's lifetime.
|
||||
type HarnessRunner struct {
|
||||
newOutput func() HarnessOutput
|
||||
}
|
||||
|
||||
// NewHarnessRunner returns a runner using newOutput to read each
|
||||
// invocation. A nil newOutput reads nothing: no tool calls, usage or session.
|
||||
func NewHarnessRunner(newOutput func() HarnessOutput) *HarnessRunner {
|
||||
return &HarnessRunner{newOutput: newOutput}
|
||||
}
|
||||
|
||||
func (r *HarnessRunner) Run(ctx context.Context, workspace string, req Request, report ToolCallReporter) (Result, error) {
|
||||
if err := validateHarness(req); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
res := Result{Version: ProtocolVersion, TaskID: req.TaskID, Attempt: req.Attempt, Stage: req.Stage}
|
||||
before, err := snapshotRepo(workspace)
|
||||
if err != nil {
|
||||
return Result{}, fmt.Errorf("snapshot workspace before harness: %w", err)
|
||||
}
|
||||
runCtx := ctx
|
||||
if req.Budget.WallClock > 0 {
|
||||
var cancel context.CancelFunc
|
||||
runCtx, cancel = context.WithTimeout(ctx, req.Budget.WallClock)
|
||||
defer cancel()
|
||||
}
|
||||
|
||||
maxIterations := 1
|
||||
if len(req.Harness.Resume) > 0 || len(req.Harness.Continue) > 0 {
|
||||
maxIterations = max(1, req.Gate.MaxConsecutiveFailures)
|
||||
}
|
||||
prompt := harnessPrompt(workspace, req)
|
||||
verb := req.Harness.Prompt
|
||||
session := ""
|
||||
for {
|
||||
res.Iterations++
|
||||
out := r.output()
|
||||
exitErr := invoke(runCtx, workspace, harnessArgv(req.Harness, verb, prompt, session), out, report)
|
||||
if id := out.SessionID(); id != "" {
|
||||
session = id
|
||||
}
|
||||
res.Usage = addUsage(res.Usage, out.Usage())
|
||||
res.TokensUsed = res.Usage.TotalTokens
|
||||
if ctx.Err() != nil {
|
||||
return res, ctx.Err()
|
||||
}
|
||||
|
||||
res, stopped, err := settle(runCtx, workspace, before, req, res, exitErr)
|
||||
if err != nil || stopped {
|
||||
return res, err
|
||||
}
|
||||
|
||||
gateOutput, gateErr := runHarnessGate(runCtx, req.Gate.Commands, workspace)
|
||||
if gateErr == nil {
|
||||
res.Status = StatusPassed
|
||||
return res, nil
|
||||
}
|
||||
if runCtx.Err() != nil {
|
||||
return park(res, StopTimedOut, "harness exceeded its wall-clock budget during the gate"), nil
|
||||
}
|
||||
if res.Iterations >= maxIterations {
|
||||
return park(res, StopGate, gateOutput), nil
|
||||
}
|
||||
verb, prompt = resumeVerb(req.Harness, session), "The quality gate failed. Fix the cause, then stop.\n\n"+gateOutput
|
||||
}
|
||||
}
|
||||
|
||||
// settle checks one finished invocation. Guarantees come first: a policy
|
||||
// violation parks the step even when the invocation also timed out or
|
||||
// failed, because what the harness did matters more than how it ended.
|
||||
func settle(runCtx context.Context, workspace string, before repoState, req Request, res Result, exitErr error) (Result, bool, error) {
|
||||
after, err := snapshotRepo(workspace)
|
||||
if err != nil {
|
||||
return res, true, fmt.Errorf("snapshot workspace after harness: %w", err)
|
||||
}
|
||||
res.Changes = changedPaths(before, after)
|
||||
// An exhausted budget is an outcome of the step, not a runner error.
|
||||
outOfTime := errors.Is(runCtx.Err(), context.DeadlineExceeded)
|
||||
switch violation := policyViolation(before, after, res.Changes, req); {
|
||||
case violation != "":
|
||||
return park(res, StopPolicy, violation), true, nil
|
||||
case outOfTime:
|
||||
return park(res, StopTimedOut, "harness exceeded its wall-clock budget"), true, nil
|
||||
case exitErr != nil:
|
||||
return park(res, StopExited, exitErr.Error()), true, nil
|
||||
}
|
||||
return res, false, nil
|
||||
}
|
||||
|
||||
func (r *HarnessRunner) output() HarnessOutput {
|
||||
if r.newOutput == nil {
|
||||
return silentOutput{}
|
||||
}
|
||||
return r.newOutput()
|
||||
}
|
||||
|
||||
func validateHarness(req Request) error {
|
||||
if err := req.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
h := req.Harness
|
||||
if h == nil || len(h.Launch) == 0 {
|
||||
return errors.New("harness runner needs a harness spec with a launch command")
|
||||
}
|
||||
if !slices.ContainsFunc(h.Prompt, func(s string) bool { return strings.Contains(s, promptPlaceholder) }) {
|
||||
return fmt.Errorf("harness prompt verb must carry %s, or the mission is never passed", promptPlaceholder)
|
||||
}
|
||||
if len(h.Resume) > 0 && !slices.ContainsFunc(h.Resume, func(s string) bool { return strings.Contains(s, sessionPlaceholder) }) {
|
||||
return fmt.Errorf("harness resume verb must carry %s", sessionPlaceholder)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// harnessArgv is the launch command, then the verb, then the prompt verb
|
||||
// when the verb is a resume or continue, with placeholders substituted as
|
||||
// raw values rather than shell text.
|
||||
func harnessArgv(h *HarnessSpec, verb []string, prompt, session string) []string {
|
||||
argv := slices.Clone(h.Launch)
|
||||
tails := [][]string{verb}
|
||||
if !slices.Equal(verb, h.Prompt) {
|
||||
tails = append(tails, h.Prompt)
|
||||
}
|
||||
for _, tail := range tails {
|
||||
for _, arg := range tail {
|
||||
arg = strings.ReplaceAll(arg, promptPlaceholder, prompt)
|
||||
arg = strings.ReplaceAll(arg, sessionPlaceholder, session)
|
||||
argv = append(argv, arg)
|
||||
}
|
||||
}
|
||||
return argv
|
||||
}
|
||||
|
||||
// resumeVerb continues the session the last invocation reported, falling
|
||||
// back to the most recent session when the CLI reported none.
|
||||
func resumeVerb(h *HarnessSpec, sessionID string) []string {
|
||||
if len(h.Resume) > 0 && sessionID != "" {
|
||||
return slices.Clone(h.Resume)
|
||||
}
|
||||
if len(h.Continue) > 0 {
|
||||
return slices.Clone(h.Continue)
|
||||
}
|
||||
return slices.Clone(h.Resume)
|
||||
}
|
||||
|
||||
// invoke runs one harness invocation in its own process group, feeding
|
||||
// stdout to out line by line. Cancelling ctx kills the whole group, so a
|
||||
// CLI's children cannot outlive the step.
|
||||
func invoke(ctx context.Context, workspace string, argv []string, out HarnessOutput, report ToolCallReporter) error {
|
||||
cmd := exec.CommandContext(ctx, argv[0], argv[1:]...)
|
||||
cmd.Dir = workspace
|
||||
cmd.WaitDelay = killGrace
|
||||
setProcessGroup(cmd)
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var stderr tailBuffer
|
||||
cmd.Stderr = &stderr
|
||||
if err := cmd.Start(); err != nil {
|
||||
return fmt.Errorf("start harness: %w", err)
|
||||
}
|
||||
scanner := bufio.NewScanner(stdout)
|
||||
scanner.Buffer(make([]byte, 64*1024), 16*1024*1024)
|
||||
for scanner.Scan() {
|
||||
out.Line(scanner.Bytes(), report)
|
||||
}
|
||||
_, _ = io.Copy(io.Discard, stdout)
|
||||
if err := cmd.Wait(); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
return fmt.Errorf("harness exited: %w: %s", err, strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func harnessPrompt(workspace string, req Request) string {
|
||||
var b strings.Builder
|
||||
b.WriteString(req.Mission)
|
||||
b.WriteString("\n\n## Rules\n\n")
|
||||
b.WriteString(projectScopedRules(workspace, req.ExtraRules))
|
||||
b.WriteString("\nDo not run git commands that change history, refs or configuration. Archie commits your changes.")
|
||||
if req.ReadOnly {
|
||||
b.WriteString("\nThis stage is read-only. Do not create, modify or delete any file.")
|
||||
}
|
||||
if patterns := slices.Concat(req.Protection.Suffixes, req.Protection.Globs); len(patterns) > 0 {
|
||||
b.WriteString("\nDo not modify files matching: " + strings.Join(patterns, ", "))
|
||||
}
|
||||
if len(req.Gate.Commands) > 0 {
|
||||
b.WriteString("\nYour work must pass these commands:")
|
||||
for _, c := range req.Gate.Commands {
|
||||
b.WriteString("\n- " + strings.Join(c.Argv, " "))
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(req.Notes) != "" {
|
||||
b.WriteString("\n\n## Notes\n\n" + req.Notes)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func runHarnessGate(ctx context.Context, commands []Command, workspace string) (string, error) {
|
||||
for _, c := range commands {
|
||||
if len(c.Argv) == 0 {
|
||||
continue
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, c.Argv[0], c.Argv[1:]...)
|
||||
cmd.Dir = workspace
|
||||
output, err := cmd.CombinedOutput()
|
||||
switch {
|
||||
case c.ExpectFailure && err == nil:
|
||||
return fmt.Sprintf("[%s] expected this command to fail, but it passed:\n%s", c.Name, clipGate(output)),
|
||||
fmt.Errorf("gate %s: expected failure", c.Name)
|
||||
case !c.ExpectFailure && err != nil:
|
||||
return fmt.Sprintf("[%s] %s\n%s", c.Name, strings.Join(c.Argv, " "), clipGate(output)), fmt.Errorf("gate %s: %w", c.Name, err)
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func clipGate(out []byte) string {
|
||||
const limit = 4000
|
||||
s := strings.TrimSpace(string(out))
|
||||
if len(s) <= limit {
|
||||
return s
|
||||
}
|
||||
return s[:limit/2] + "\n...[truncated]...\n" + s[len(s)-limit/2:]
|
||||
}
|
||||
|
||||
func park(res Result, stop, detail string) Result {
|
||||
res.Status, res.StopReason, res.Detail = StatusParked, stop, detail
|
||||
return res
|
||||
}
|
||||
|
||||
func addUsage(a, b Usage) Usage {
|
||||
return Usage{
|
||||
PromptTokens: a.PromptTokens + b.PromptTokens,
|
||||
CompletionTokens: a.CompletionTokens + b.CompletionTokens,
|
||||
TotalTokens: a.TotalTokens + b.TotalTokens,
|
||||
CachedTokens: a.CachedTokens + b.CachedTokens,
|
||||
CacheCreationTokens: a.CacheCreationTokens + b.CacheCreationTokens,
|
||||
}
|
||||
}
|
||||
|
||||
// repoState is what the runner compares across an invocation: every ref
|
||||
// (HEAD included), the repository configuration, and a content hash of each
|
||||
// path git reports as not clean.
|
||||
type repoState struct {
|
||||
refs map[string]string
|
||||
config string
|
||||
dirty map[string]string
|
||||
}
|
||||
|
||||
func snapshotRepo(workspace string) (repoState, error) {
|
||||
repo, err := git.PlainOpen(workspace)
|
||||
if err != nil {
|
||||
return repoState{}, err
|
||||
}
|
||||
state := repoState{refs: map[string]string{}, dirty: map[string]string{}}
|
||||
if head, err := repo.Storer.Reference(plumbing.HEAD); err == nil {
|
||||
state.refs[plumbing.HEAD.String()] = head.String()
|
||||
}
|
||||
iter, err := repo.References()
|
||||
if err != nil {
|
||||
return repoState{}, err
|
||||
}
|
||||
if err := iter.ForEach(func(ref *plumbing.Reference) error {
|
||||
state.refs[ref.Name().String()] = ref.String()
|
||||
return nil
|
||||
}); err != nil {
|
||||
return repoState{}, err
|
||||
}
|
||||
cfg, err := repo.Config()
|
||||
if err != nil {
|
||||
return repoState{}, err
|
||||
}
|
||||
raw, err := cfg.Marshal()
|
||||
if err != nil {
|
||||
return repoState{}, err
|
||||
}
|
||||
state.config = string(raw)
|
||||
wt, err := repo.Worktree()
|
||||
if err != nil {
|
||||
return repoState{}, err
|
||||
}
|
||||
status, err := wt.Status()
|
||||
if err != nil {
|
||||
return repoState{}, err
|
||||
}
|
||||
for path, st := range status {
|
||||
if st.Worktree == git.Unmodified && st.Staging == git.Unmodified {
|
||||
continue
|
||||
}
|
||||
state.dirty[path] = contentHash(filepath.Join(workspace, path))
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
|
||||
func contentHash(path string) string {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "absent"
|
||||
}
|
||||
sum := sha256.Sum256(data)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// changedPaths is every path whose state differs across the invocation. A
|
||||
// path already dirty and left untouched is not a change of this run.
|
||||
func changedPaths(before, after repoState) []string {
|
||||
var changed []string
|
||||
for path := range mergeKeys(before.dirty, after.dirty) {
|
||||
if before.dirty[path] != after.dirty[path] {
|
||||
changed = append(changed, path)
|
||||
}
|
||||
}
|
||||
slices.Sort(changed)
|
||||
return changed
|
||||
}
|
||||
|
||||
func mergeKeys(a, b map[string]string) map[string]struct{} {
|
||||
keys := make(map[string]struct{}, len(a)+len(b))
|
||||
for k := range maps.Keys(a) {
|
||||
keys[k] = struct{}{}
|
||||
}
|
||||
for k := range maps.Keys(b) {
|
||||
keys[k] = struct{}{}
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
func policyViolation(before, after repoState, changes []string, req Request) string {
|
||||
if !maps.Equal(before.refs, after.refs) {
|
||||
var moved []string
|
||||
for name := range mergeKeys(before.refs, after.refs) {
|
||||
if before.refs[name] != after.refs[name] {
|
||||
moved = append(moved, name)
|
||||
}
|
||||
}
|
||||
slices.Sort(moved)
|
||||
return "the harness changed git refs: " + strings.Join(moved, ", ")
|
||||
}
|
||||
if before.config != after.config {
|
||||
return "the harness changed the git configuration"
|
||||
}
|
||||
if req.ReadOnly && len(changes) > 0 {
|
||||
return "the harness wrote in a read-only stage: " + strings.Join(changes, ", ")
|
||||
}
|
||||
if protected := protectionMatcher(req.Protection, false); protected != nil {
|
||||
var hit []string
|
||||
for _, path := range changes {
|
||||
if protected(path) {
|
||||
hit = append(hit, path)
|
||||
}
|
||||
}
|
||||
if len(hit) > 0 {
|
||||
return "the harness modified protected paths: " + strings.Join(hit, ", ")
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type silentOutput struct{}
|
||||
|
||||
func (silentOutput) Line([]byte, ToolCallReporter) {}
|
||||
func (silentOutput) SessionID() string { return "" }
|
||||
func (silentOutput) Usage() Usage { return Usage{} }
|
||||
|
||||
// tailBuffer keeps the last stderrTailBytes written to it.
|
||||
type tailBuffer struct{ buf bytes.Buffer }
|
||||
|
||||
func (t *tailBuffer) Write(p []byte) (int, error) {
|
||||
t.buf.Write(p)
|
||||
if extra := t.buf.Len() - stderrTailBytes; extra > 0 {
|
||||
t.buf.Next(extra)
|
||||
}
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func (t *tailBuffer) String() string { return t.buf.String() }
|
||||
@@ -0,0 +1,289 @@
|
||||
package agentexec
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// fakeHarness is a stand-in coding-agent CLI. FAKE_SCENARIO picks what it
|
||||
// does to the workspace; every invocation's arguments are appended to
|
||||
// FAKE_LOG, and it prints a session line and a tool line for the adapter.
|
||||
const fakeHarness = `#!/bin/sh
|
||||
printf '%s' "$*" | tr '\n' ' ' >> "$FAKE_LOG"
|
||||
echo >> "$FAKE_LOG"
|
||||
case "$FAKE_SCENARIO" in
|
||||
edit) echo hello > notes.txt ;;
|
||||
commit) echo x > a.txt; git add a.txt; git -c user.email=a@b.c -c user.name=a commit -qm sneaky ;;
|
||||
config) git config remote.origin.url https://attacker.invalid/repo.git ;;
|
||||
protected) echo broken >> widget_test.go ;;
|
||||
sleep) sleep 30 & echo $! > "$FAKE_LOG.child"; wait ;;
|
||||
fix-on-resume) if [ -f .attempted ]; then echo ok > fixed.txt; else touch .attempted; fi ;;
|
||||
esac
|
||||
echo "session=S123"
|
||||
echo "tool=edit_file"
|
||||
`
|
||||
|
||||
// lineOutput is a test adapter over fakeHarness's output lines.
|
||||
type lineOutput struct {
|
||||
mu sync.Mutex
|
||||
session string
|
||||
}
|
||||
|
||||
func (o *lineOutput) Line(line []byte, report ToolCallReporter) {
|
||||
text := string(line)
|
||||
switch {
|
||||
case strings.HasPrefix(text, "session="):
|
||||
o.mu.Lock()
|
||||
o.session = strings.TrimPrefix(text, "session=")
|
||||
o.mu.Unlock()
|
||||
case strings.HasPrefix(text, "tool=") && report != nil:
|
||||
report(ToolCallReport{Tool: strings.TrimPrefix(text, "tool=")})
|
||||
}
|
||||
}
|
||||
|
||||
func (o *lineOutput) SessionID() string {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
return o.session
|
||||
}
|
||||
|
||||
func (o *lineOutput) Usage() Usage { return Usage{} }
|
||||
|
||||
func runGit(t *testing.T, dir string, args ...string) {
|
||||
t.Helper()
|
||||
cmd := exec.CommandContext(t.Context(), "git", append([]string{"-c", "user.email=a@b.c", "-c", "user.name=a"}, args...)...)
|
||||
cmd.Dir = dir
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
t.Fatalf("git %v: %v\n%s", args, err, out)
|
||||
}
|
||||
}
|
||||
|
||||
type fixture struct {
|
||||
workspace string
|
||||
log string
|
||||
req Request
|
||||
}
|
||||
|
||||
func newFixture(t *testing.T, scenario string) *fixture {
|
||||
t.Helper()
|
||||
ws := t.TempDir()
|
||||
runGit(t, ws, "init", "-q")
|
||||
if err := os.WriteFile(filepath.Join(ws, "widget_test.go"), []byte("package widget\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runGit(t, ws, "add", ".")
|
||||
runGit(t, ws, "commit", "-qm", "base")
|
||||
runGit(t, ws, "remote", "add", "origin", "https://forge.invalid/repo.git")
|
||||
|
||||
bin := filepath.Join(t.TempDir(), "fake-harness")
|
||||
if err := os.WriteFile(bin, []byte(fakeHarness), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
log := filepath.Join(t.TempDir(), "invocations")
|
||||
t.Setenv("FAKE_SCENARIO", scenario)
|
||||
t.Setenv("FAKE_LOG", log)
|
||||
return &fixture{workspace: ws, log: log, req: Request{
|
||||
Version: ProtocolVersion, TaskID: 1, Attempt: 1, Stage: "implement", Model: "harness",
|
||||
Mission: "Write the notes file.",
|
||||
Budget: Budget{WallClock: 20 * time.Second},
|
||||
Harness: &HarnessSpec{
|
||||
Launch: []string{"/bin/sh", bin},
|
||||
Prompt: []string{"-p", "{{.Prompt}}"},
|
||||
Resume: []string{"--resume", "{{.SessionID}}"},
|
||||
},
|
||||
}}
|
||||
}
|
||||
|
||||
func (f *fixture) run(t *testing.T, ctx context.Context) (Result, []ToolCallReport) {
|
||||
t.Helper()
|
||||
var mu sync.Mutex
|
||||
var calls []ToolCallReport
|
||||
res, err := NewHarnessRunner(func() HarnessOutput { return &lineOutput{} }).Run(ctx, f.workspace, f.req, func(r ToolCallReport) {
|
||||
mu.Lock()
|
||||
calls = append(calls, r)
|
||||
mu.Unlock()
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
return res, calls
|
||||
}
|
||||
|
||||
func (f *fixture) invocations(t *testing.T) []string {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(f.log)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
sc := bufio.NewScanner(bytes.NewReader(raw))
|
||||
for sc.Scan() {
|
||||
out = append(out, sc.Text())
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestHarnessRunsTheMissionAndReportsChanges(t *testing.T) {
|
||||
f := newFixture(t, "edit")
|
||||
res, calls := f.run(t, t.Context())
|
||||
if res.Status != StatusPassed {
|
||||
t.Fatalf("status %q (%s), want passed", res.Status, res.StopReason)
|
||||
}
|
||||
if !slices.Equal(res.Changes, []string{"notes.txt"}) {
|
||||
t.Fatalf("changes %v, want [notes.txt]", res.Changes)
|
||||
}
|
||||
if len(calls) != 1 || calls[0].Tool != "edit_file" {
|
||||
t.Fatalf("tool calls %v, want one edit_file", calls)
|
||||
}
|
||||
inv := f.invocations(t)
|
||||
if len(inv) != 1 || !strings.HasPrefix(inv[0], "-p ") || !strings.Contains(inv[0], "Write the notes file.") {
|
||||
t.Fatalf("invocations %q, want one headless prompt carrying the mission", inv)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHarnessGuaranteesHoldWhateverTheHarnessDoes(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
scenario string
|
||||
setup func(*Request)
|
||||
want string
|
||||
}{
|
||||
{"a commit fails the step", "commit", nil, "git"},
|
||||
{"rewriting the git config fails the step", "config", nil, "git"},
|
||||
{"editing a protected path fails the step", "protected", func(r *Request) { r.Protection = Protection{Suffixes: []string{"_test.go"}} }, "widget_test.go"},
|
||||
{"writing in a read-only stage fails the step", "edit", func(r *Request) { r.ReadOnly = true }, "notes.txt"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
f := newFixture(t, tt.scenario)
|
||||
if tt.setup != nil {
|
||||
tt.setup(&f.req)
|
||||
}
|
||||
res, _ := f.run(t, t.Context())
|
||||
if res.Status != StatusParked || !strings.Contains(res.StopReason, StopPolicy) || !strings.Contains(res.Detail, tt.want) {
|
||||
t.Fatalf("status %q stop %q detail %q; want parked on policy naming %q", res.Status, res.StopReason, res.Detail, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHarnessIgnoresChangesThatPredateTheRun(t *testing.T) {
|
||||
f := newFixture(t, "edit")
|
||||
if err := os.WriteFile(filepath.Join(f.workspace, "earlier.txt"), []byte("from a previous stage\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.req.ReadOnly = false
|
||||
res, _ := f.run(t, t.Context())
|
||||
if !slices.Equal(res.Changes, []string{"notes.txt"}) {
|
||||
t.Fatalf("changes %v, want only what this run changed", res.Changes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHarnessBudgetAndCancellation(t *testing.T) {
|
||||
t.Run("the wall clock kills the harness", func(t *testing.T) {
|
||||
f := newFixture(t, "sleep")
|
||||
f.req.Budget.WallClock = 500 * time.Millisecond
|
||||
start := time.Now()
|
||||
res, _ := f.run(t, t.Context())
|
||||
if time.Since(start) > 10*time.Second {
|
||||
t.Fatalf("run took %s; the harness outlived its budget", time.Since(start))
|
||||
}
|
||||
if res.Status != StatusParked || res.StopReason != StopTimedOut {
|
||||
t.Fatalf("status %q stop %q, want parked timed_out", res.Status, res.StopReason)
|
||||
}
|
||||
assertChildDead(t, f.log+".child")
|
||||
})
|
||||
t.Run("cancelling the run kills the harness", func(t *testing.T) {
|
||||
f := newFixture(t, "sleep")
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
time.AfterFunc(300*time.Millisecond, cancel)
|
||||
start := time.Now()
|
||||
_, err := NewHarnessRunner(func() HarnessOutput { return &lineOutput{} }).Run(ctx, f.workspace, f.req, nil)
|
||||
if time.Since(start) > 10*time.Second {
|
||||
t.Fatalf("cancel took %s", time.Since(start))
|
||||
}
|
||||
if err == nil || ctx.Err() == nil {
|
||||
t.Fatalf("Run after cancel returned %v, want the cancellation", err)
|
||||
}
|
||||
assertChildDead(t, f.log+".child")
|
||||
})
|
||||
}
|
||||
|
||||
func TestHarnessGateResumesTheSession(t *testing.T) {
|
||||
gate := Gate{Commands: []Command{{Name: "fixed", Argv: []string{"test", "-f", "fixed.txt"}}}, MaxConsecutiveFailures: 3}
|
||||
|
||||
t.Run("a failed gate resumes the same session with its output", func(t *testing.T) {
|
||||
f := newFixture(t, "fix-on-resume")
|
||||
f.req.Gate = gate
|
||||
res, _ := f.run(t, t.Context())
|
||||
if res.Status != StatusPassed || res.Iterations != 2 {
|
||||
t.Fatalf("status %q after %d iterations (%s), want passed after 2", res.Status, res.Iterations, res.StopReason)
|
||||
}
|
||||
inv := f.invocations(t)
|
||||
if len(inv) != 2 || !strings.HasPrefix(inv[1], "--resume S123 -p ") || !strings.Contains(inv[1], "[fixed]") {
|
||||
t.Fatalf("invocations %q, want a resume of S123 carrying the gate failure", inv)
|
||||
}
|
||||
})
|
||||
t.Run("a gate that never passes parks after the cap", func(t *testing.T) {
|
||||
f := newFixture(t, "edit")
|
||||
f.req.Gate = gate
|
||||
res, _ := f.run(t, t.Context())
|
||||
if res.Status != StatusParked || res.Iterations != 3 {
|
||||
t.Fatalf("status %q after %d iterations, want parked after 3", res.Status, res.Iterations)
|
||||
}
|
||||
})
|
||||
t.Run("without a resume verb there is one iteration", func(t *testing.T) {
|
||||
f := newFixture(t, "fix-on-resume")
|
||||
f.req.Gate = gate
|
||||
f.req.Harness.Resume = nil
|
||||
res, _ := f.run(t, t.Context())
|
||||
if res.Status != StatusParked || res.Iterations != 1 || len(f.invocations(t)) != 1 {
|
||||
t.Fatalf("status %q after %d iterations and %d invocations, want parked after 1", res.Status, res.Iterations, len(f.invocations(t)))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestHarnessRequestValidation(t *testing.T) {
|
||||
f := newFixture(t, "edit")
|
||||
f.req.Harness.Prompt = []string{"-p"}
|
||||
if _, err := NewHarnessRunner(nil).Run(t.Context(), f.workspace, f.req, nil); err == nil {
|
||||
t.Fatal("a prompt verb with no {{.Prompt}} placeholder ran; the mission would be silently dropped")
|
||||
}
|
||||
f.req.Harness = nil
|
||||
if _, err := NewHarnessRunner(nil).Run(t.Context(), f.workspace, f.req, nil); err == nil {
|
||||
t.Fatal("a request without a harness spec ran on the harness runner")
|
||||
}
|
||||
}
|
||||
|
||||
// assertChildDead checks the harness's own child process did not outlive
|
||||
// it: killing only the CLI would leave its subprocesses running.
|
||||
func assertChildDead(t *testing.T, pidFile string) {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(pidFile)
|
||||
if err != nil {
|
||||
t.Fatalf("the harness recorded no child: %v", err)
|
||||
}
|
||||
pid, err := strconv.Atoi(strings.TrimSpace(string(raw)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for syscall.Kill(pid, 0) == nil {
|
||||
if time.Now().After(deadline) {
|
||||
_ = syscall.Kill(pid, syscall.SIGKILL)
|
||||
t.Fatalf("harness child %d outlived the step", pid)
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
@@ -88,6 +88,20 @@ type Request struct {
|
||||
// directory. Each entry carries the name and source so the agent
|
||||
// can register them as tools. PRD section 5 Layer 1.
|
||||
Plugins []PluginSpec `json:"plugins,omitempty"`
|
||||
// Harness, when set, runs the stage on an external coding-agent CLI
|
||||
// instead of the built-in loop. The worker in a Kit container is the
|
||||
// daemon's own archie-agent build, so it always understands this field.
|
||||
Harness *HarnessSpec `json:"harness,omitempty"`
|
||||
}
|
||||
|
||||
// HarnessSpec is how to drive a Kit's agent headlessly: its launch argv
|
||||
// (image entrypoint and cmd) and the agent-sessions@1 verb tails appended
|
||||
// to it. Prompt must carry {{.Prompt}}; Resume, when set, {{.SessionID}}.
|
||||
type HarnessSpec struct {
|
||||
Launch []string `json:"launch"`
|
||||
Prompt []string `json:"prompt"`
|
||||
Resume []string `json:"resume,omitempty"`
|
||||
Continue []string `json:"continue,omitempty"`
|
||||
}
|
||||
|
||||
// PluginSpec is a bundled Yaegi plugin passed from daemon to agent.
|
||||
|
||||
Reference in new issue
Block a user