Files
archie/internal/domain/workflow/agent_test.go
T
sam 20bff1a367 feat(agentexec,workflow): surface tool calls on the task timeline
archie-core-467 wired tool-call visibility into the interactive chat
gateway, but a workflow task's own transcript never showed what its
tools actually did -- only stage/agent-finish events existed, so
there was no way to confirm an agent's summary matched what it really
ran.

Adds a ToolCallReporter, invoked once per completed call from
toolExecute (the single choke point every tool call passes through
before reaching a model), reported via a new ToolSetOptions.OnToolCall
hook. Runner.Run grows a report parameter to carry it in: InProcessRunner
wires it through, since it executes in the same process as its tools;
SubprocessRunner and NATSRunner accept and ignore it, since their tool
calls happen in a different process with nothing here to observe them.

The workflow engine turns each report into a tool_call event on the
task's own Bus (agent.go, implement.go), riding the same NATS
event-forwarding archie-core-518 already built to get container
events onto the daemon's timeline.

Scoped to the central tool registry (read/write/edit/grep/shell/...)
built via BuildToolSet; capture tools (structured finish output) and
run_go_script build their ToolSet entries directly and are not yet
covered.
2026-08-15 12:53:41 +10:00

420 lines
15 KiB
Go

package workflow
import (
"context"
"encoding/json"
"errors"
"log/slog"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
"github.com/samcharles93/archie-core/internal/agentexec"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/events"
"github.com/samcharles93/archie-core/internal/store"
)
type fakeAgentRunner struct {
workspace string
request agentexec.Request
result agentexec.Result
err error
// report is the reporter the caller passed to Run, captured so a test
// can invoke it directly and assert on the resulting tc.Emit call.
report agentexec.ToolCallReporter
}
type agentRunnerFunc func(context.Context, string, agentexec.Request, agentexec.ToolCallReporter) (agentexec.Result, error)
func (f agentRunnerFunc) Run(
ctx context.Context, workspace string, req agentexec.Request, report agentexec.ToolCallReporter,
) (agentexec.Result, error) {
return f(ctx, workspace, req, report)
}
func (r *fakeAgentRunner) Run(
_ context.Context, workspace string, req agentexec.Request, report agentexec.ToolCallReporter,
) (agentexec.Result, error) {
r.workspace = workspace
r.request = req
r.report = report
if r.result.Version == 0 {
r.result.Version = agentexec.ProtocolVersion
}
r.result.TaskID = req.TaskID
r.result.Attempt = req.Attempt
r.result.Stage = req.Stage
return r.result, r.err
}
func TestAgentStageBuildsExecutionRequestAndAppliesResult(t *testing.T) {
runner := &fakeAgentRunner{result: agentexec.Result{
Version: agentexec.ProtocolVersion, Status: agentexec.StatusPassed,
Summary: "finished", TokensUsed: 23, Iterations: 4,
}}
resultCalled := false
stage := AgentStage{
Name: "plan", Role: "planner", ReadOnly: true, MaxSteps: 7,
Mission: func(*TaskContext) string { return "inspect the repository" },
OnResult: func(_ *TaskContext, result agentexec.Result) error {
resultCalled = result.Summary == "finished"
return nil
},
}.Stage()
task := &store.Task{ID: 9, Attempt: 2}
tc := &TaskContext{
Task: task, Agent: runner, Dir: "/tmp/workspace", Log: slog.New(slog.DiscardHandler),
Cfg: config.Config{
Models: map[string]string{"builder": "provider/fallback"},
Budgets: config.Budgets{MaxSteps: 50, MaxTokens: 5000},
},
Repo: config.Repo{Protect: []string{"_templ.go"}, Preflight: [][]string{{"go", "version"}}},
}
if err := stage.Run(context.Background(), tc); err != nil {
t.Fatal(err)
}
if runner.workspace != tc.Dir {
t.Fatalf("workspace = %q, want %q", runner.workspace, tc.Dir)
}
wantPreflight := []agentexec.Command{{Name: "go", Argv: []string{"go", "version"}}}
if runner.request.Version != agentexec.ProtocolVersion || runner.request.TaskID != 9 || runner.request.Attempt != 2 ||
runner.request.Model != "provider/fallback" || runner.request.Budget.MaxSteps != 7 || !runner.request.ReadOnly ||
!reflect.DeepEqual(runner.request.Preflight, wantPreflight) {
t.Fatalf("unexpected request: %#v", runner.request)
}
if len(runner.request.Protection.Suffixes) != 0 {
t.Fatalf("read-only request has protection rules: %#v", runner.request.Protection)
}
if !resultCalled || task.TokensUsed != 23 || task.Iterations != 4 {
t.Fatalf("result not applied: called=%v task=%#v", resultCalled, task)
}
}
// TestAgentStageReportsToolCallsOnTheTaskBus pins archie-core-467's
// task-transcript counterpart: the reporter AgentStage.Stage() passes into
// tc.Agent.Run must, when invoked, publish a tool_call event onto tc.Bus
// carrying the tool name and outcome -- otherwise a workflow task's own
// transcript never shows what its tools actually did, only the chat
// gateway's interactive turns do.
func TestAgentStageReportsToolCallsOnTheTaskBus(t *testing.T) {
runner := &fakeAgentRunner{result: agentexec.Result{
Version: agentexec.ProtocolVersion, Status: agentexec.StatusPassed,
}}
stage := AgentStage{
Name: "build",
Role: "builder",
Mission: func(*TaskContext) string { return "implement the feature" },
}.Stage()
bus := events.NewBus()
sub := bus.Subscribe(8)
t.Cleanup(sub.Close)
task := &store.Task{ID: 9, Owner: "acme", Repo: "widget", Attempt: 1}
tc := &TaskContext{
Task: task, Agent: runner, Dir: "/tmp/workspace", Bus: bus, Log: slog.New(slog.DiscardHandler),
Cfg: config.Config{Models: map[string]string{"builder": "provider/model"}},
}
if err := stage.Run(context.Background(), tc); err != nil {
t.Fatal(err)
}
if runner.report == nil {
t.Fatal("AgentStage did not pass a ToolCallReporter to Run")
}
runner.report(agentexec.ToolCallReport{Tool: "read_file", Detail: `"package main"`, Failed: false})
runner.report(agentexec.ToolCallReport{Tool: "shell", Detail: "error: exit status 1", Failed: true})
var got []events.Event
for len(got) < 2 {
select {
case e := <-sub.C:
if e.Kind == events.KindToolCall {
got = append(got, e)
}
case <-time.After(time.Second):
t.Fatalf("timed out waiting for tool_call events, got %d", len(got))
}
}
if got[0].Stage != "build" || got[0].Data["tool"] != "read_file" || got[0].Detail != `"package main"` || got[0].Data["failed"] != false {
t.Fatalf("first tool_call event = %#v", got[0])
}
if got[1].Data["tool"] != "shell" || got[1].Detail != "error: exit status 1" || got[1].Data["failed"] != true {
t.Fatalf("second tool_call event = %#v", got[1])
}
}
func TestAgentStagePersistsReturnedNotes(t *testing.T) {
st, err := store.Open(t.Context(), filepath.Join(t.TempDir(), "archie.db"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
if err := st.Close(); err != nil {
t.Errorf("close store: %v", err)
}
})
ctx := context.Background()
if _, err := st.EnqueueIssue(ctx, "owner", "repo", 1, "title", "body", "", ""); err != nil {
t.Fatal(err)
}
task, err := st.ClaimNext(ctx)
if err != nil {
t.Fatal(err)
}
runner := &fakeAgentRunner{result: agentexec.Result{
Version: agentexec.ProtocolVersion, Status: agentexec.StatusPassed,
AppendedNotes: []string{"checked with go test"},
}}
stage := AgentStage{
Name: "build", Role: "builder", Mission: func(*TaskContext) string { return "build" },
}.Stage()
tc := &TaskContext{
Task: task, Store: st, Agent: runner, Log: slog.New(slog.DiscardHandler),
Cfg: config.Config{Models: map[string]string{"builder": "provider/model"}},
}
if err := stage.Run(ctx, tc); err != nil {
t.Fatal(err)
}
if task.Notes != "- checked with go test\n" {
t.Fatalf("task notes = %q", task.Notes)
}
}
func TestAgentStageDiscardsUnstampedErrorResult(t *testing.T) {
task := &store.Task{ID: 1, Attempt: 1, Notes: "existing\n"}
runner := agentRunnerFunc(func(context.Context, string, agentexec.Request, agentexec.ToolCallReporter) (agentexec.Result, error) {
return agentexec.Result{AppendedNotes: []string{"untrusted"}}, errors.New("worker exited")
})
stage := AgentStage{
Name: "build", Role: "builder", Mission: func(*TaskContext) string { return "build" },
}.Stage()
err := stage.Run(context.Background(), &TaskContext{
Task: task, Agent: runner, Log: slog.New(slog.DiscardHandler),
Cfg: config.Config{Models: map[string]string{"builder": "provider/model"}},
})
if err == nil {
t.Fatal("stage accepted an unstamped error result")
}
if task.Notes != "existing\n" {
t.Fatalf("unstamped error result mutated notes: %q", task.Notes)
}
}
func TestFeasibilityDecisionCrossesAsCapturedData(t *testing.T) {
runner := &fakeAgentRunner{result: agentexec.Result{
Version: agentexec.ProtocolVersion, Status: agentexec.StatusPassed,
Captures: map[string][]json.RawMessage{"decide": {json.RawMessage(`{"fit":true,"reasons":"aligned"}`)}},
}}
tc := &TaskContext{
Task: &store.Task{ID: 1}, Agent: runner, Log: slog.New(slog.DiscardHandler),
Cfg: config.Config{Models: map[string]string{"planner": "provider/model"}},
Repo: config.Repo{Owner: "owner", Name: "repo"},
}
if err := Feasibility().Stages[1].Run(context.Background(), tc); err != nil {
t.Fatal(err)
}
if tc.decision == nil || !tc.decision.Fit || tc.decision.Reasons != "aligned" {
t.Fatalf("decision = %#v", tc.decision)
}
}
func TestImplementPlanTreatsChatSourceAsNativeTask(t *testing.T) {
runner := &fakeAgentRunner{result: agentexec.Result{
Status: agentexec.StatusPassed, Summary: "plan",
}}
forgeClient := &fakeForge{}
tc := &TaskContext{
Task: &store.Task{
ID: 1, IssueNumber: 999_001, Title: "Fix chat task", Source: store.SourceChat,
},
Agent: runner,
Forge: forgeClient,
Log: slog.New(slog.DiscardHandler),
Cfg: config.Config{Models: map[string]string{"planner": "provider/model"}},
Repo: config.Repo{Owner: "owner", Name: "repo"},
}
if err := Implement().Stages[3].Run(context.Background(), tc); err != nil {
t.Fatal(err)
}
if strings.Contains(runner.request.Mission, "999001") ||
!strings.Contains(runner.request.Mission, `<task source="chat">`) {
t.Errorf("chat mission uses synthetic issue semantics:\n%s", runner.request.Mission)
}
if len(forgeClient.commented) != 0 {
t.Fatalf("chat planning posted %d issue comments", len(forgeClient.commented))
}
}
func TestFeasibilityChatTaskAvoidsIssueOperations(t *testing.T) {
t.Run("not fit", func(t *testing.T) {
runner := &fakeAgentRunner{result: agentexec.Result{
Status: agentexec.StatusPassed,
Captures: map[string][]json.RawMessage{
"decide": {json.RawMessage(`{"fit":false,"reasons":"out of scope"}`)},
},
}}
forgeClient := &fakeForge{}
tc := &TaskContext{
Task: &store.Task{
ID: 1, IssueNumber: 999_001, Title: "Feature", Source: store.SourceChat,
},
Agent: runner,
Forge: forgeClient,
Log: slog.New(slog.DiscardHandler),
Cfg: config.Config{Models: map[string]string{"planner": "provider/model"}},
Repo: config.Repo{Owner: "owner", Name: "repo"},
}
if err := Feasibility().Stages[1].Run(context.Background(), tc); err != nil {
t.Fatal(err)
}
if forgeClient.closed != 0 || tc.Outcome.Status != store.StatusClosedWontDo {
t.Fatalf("close calls/outcome = %d/%q", forgeClient.closed, tc.Outcome.Status)
}
})
t.Run("await approval", func(t *testing.T) {
forgeClient := &fakeForge{}
tc := &TaskContext{
Task: &store.Task{
ID: 1, IssueNumber: 999_001, Plan: "PRD", Source: store.SourceChat,
},
Forge: forgeClient,
Log: slog.New(slog.DiscardHandler),
}
if err := Feasibility().Stages[3].Run(context.Background(), tc); err != nil {
t.Fatal(err)
}
if len(forgeClient.commented) != 0 || tc.Task.WatchCommentID != 0 ||
tc.Outcome.Status != store.StatusWaitingHuman {
t.Fatalf("comments/watch/outcome = %d/%d/%q",
len(forgeClient.commented), tc.Task.WatchCommentID, tc.Outcome.Status)
}
})
}
func TestFeasibilityRejectsNullFitValue(t *testing.T) {
runner := &fakeAgentRunner{result: agentexec.Result{
Status: agentexec.StatusPassed,
Captures: map[string][]json.RawMessage{"decide": {json.RawMessage(`{"fit":null,"reasons":"not valid"}`)}},
}}
tc := &TaskContext{
Task: &store.Task{ID: 1}, Agent: runner, Log: slog.New(slog.DiscardHandler),
Cfg: config.Config{Models: map[string]string{"planner": "provider/model"}},
Repo: config.Repo{Owner: "owner", Name: "repo"},
}
if err := Feasibility().Stages[1].Run(context.Background(), tc); err == nil {
t.Fatal("assess accepted a null fit value")
}
if tc.decision != nil {
t.Fatalf("decision mutated from invalid capture: %#v", tc.decision)
}
}
// ── regression: Gap 5 -- daemon review step ──────────────────────────
func TestReviewResultBlocksOnResultWhenRejected(t *testing.T) {
// Gap 5: no daemon review step before human delivery.
// PRD section 1: the daemon reviews agent responses before forwarding
// to human channels. When ReviewResult returns an error, the stage
// must fail and OnResult must NOT be called. When ReviewResult
// returns nil, OnResult IS called. Currently ReviewResult exists
// but no workflow uses it -- the review step is never exercised.
runner := &fakeAgentRunner{result: agentexec.Result{
Version: agentexec.ProtocolVersion, Status: agentexec.StatusPassed,
Summary: "sensitive output", TokensUsed: 1,
}}
// Test 1: ReviewResult rejects → OnResult must not be called.
onResultCalled := false
rejectStage := AgentStage{
Name: "plan",
Role: "planner",
Mission: func(*TaskContext) string { return "test" },
ReviewResult: func(_ *TaskContext, _ agentexec.Result) error {
return agentexec.ErrBlocked
},
OnResult: func(_ *TaskContext, _ agentexec.Result) error {
onResultCalled = true
return nil
},
}.Stage()
tc := &TaskContext{
Task: &store.Task{ID: 1, Attempt: 1},
Agent: runner, Log: slog.New(slog.DiscardHandler),
Cfg: config.Config{Models: map[string]string{"planner": "provider/model"}},
}
err := rejectStage.Run(context.Background(), tc)
if err == nil {
t.Error("Gap 5: ReviewResult returned an error but the stage did not fail. " +
"ReviewResult must block the stage when it rejects output.")
}
if onResultCalled {
t.Error("Gap 5: ReviewResult rejected the output but OnResult was still called. " +
"ReviewResult must gate OnResult -- rejected output must not reach human channels.")
}
// Test 2: ReviewResult approves → OnResult IS called.
onResultCalled = false
approveStage := AgentStage{
Name: "plan",
Role: "planner",
Mission: func(*TaskContext) string { return "test" },
ReviewResult: func(_ *TaskContext, _ agentexec.Result) error {
return nil // approved
},
OnResult: func(_ *TaskContext, _ agentexec.Result) error {
onResultCalled = true
return nil
},
}.Stage()
if err := approveStage.Run(context.Background(), tc); err != nil {
t.Fatalf("unexpected stage error: %v", err)
}
if !onResultCalled {
t.Error("Gap 5: ReviewResult approved the output but OnResult was not called. " +
"Approved output must flow through to human channels.")
}
}
func TestRunLeavesInterruptedTaskForCrashRecovery(t *testing.T) {
st, err := store.Open(t.Context(), filepath.Join(t.TempDir(), "archie.db"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
if err := st.Close(); err != nil {
t.Errorf("close store: %v", err)
}
})
if _, err := st.EnqueueIssue(context.Background(), "owner", "repo", 2, "title", "body", "", ""); err != nil {
t.Fatal(err)
}
task, err := st.ClaimNext(context.Background())
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
Run(ctx, Workflow{Name: "test", Stages: []Stage{{
Name: "agent", Run: func(ctx context.Context, _ *TaskContext) error { return ctx.Err() },
}}}, &TaskContext{
Task: task, Store: st, Repo: config.Repo{Owner: "owner", Name: "repo"},
Log: slog.New(slog.DiscardHandler),
})
got, err := st.TaskByIssue(context.Background(), "owner", "repo", 2)
if err != nil {
t.Fatal(err)
}
if got.Status != store.StatusRunning || got.ParkReason != "" {
t.Fatalf("interrupted task status=%q park_reason=%q, want running with no park reason", got.Status, got.ParkReason)
}
}