feat(agentexec,workflow): surface tool calls on the task timeline
archie-core-467 wired tool-call visibility into the interactive chat gateway, but a workflow task's own transcript never showed what its tools actually did -- only stage/agent-finish events existed, so there was no way to confirm an agent's summary matched what it really ran. Adds a ToolCallReporter, invoked once per completed call from toolExecute (the single choke point every tool call passes through before reaching a model), reported via a new ToolSetOptions.OnToolCall hook. Runner.Run grows a report parameter to carry it in: InProcessRunner wires it through, since it executes in the same process as its tools; SubprocessRunner and NATSRunner accept and ignore it, since their tool calls happen in a different process with nothing here to observe them. The workflow engine turns each report into a tool_call event on the task's own Bus (agent.go, implement.go), riding the same NATS event-forwarding archie-core-518 already built to get container events onto the daemon's timeline. Scoped to the central tool registry (read/write/edit/grep/shell/...) built via BuildToolSet; capture tools (structured finish output) and run_go_script build their ToolSet entries directly and are not yet covered.
This commit is contained in:
1 parent
8f73c1febb
commit
39d8b067ec
18 files changed
+250
-42
No files matched your search
@@ -19,7 +19,10 @@ import (
|
||||
|
||||
// Runner executes one autonomous stage against an already prepared workspace.
|
||||
type Runner interface {
|
||||
Run(ctx context.Context, workspace string, req Request) (Result, error)
|
||||
// report, when non-nil, is notified once per completed tool call. Only
|
||||
// a runner executing in the same process as its tools (InProcessRunner)
|
||||
// can populate it; others accept and ignore it.
|
||||
Run(ctx context.Context, workspace string, req Request, report ToolCallReporter) (Result, error)
|
||||
}
|
||||
|
||||
type loopFunc func(context.Context, agentloop.Config) (agentloop.Result, error)
|
||||
@@ -62,7 +65,7 @@ func NewInProcessRunner(
|
||||
// it is not configurable.
|
||||
const workspaceToolset = "workspace"
|
||||
|
||||
func (r *InProcessRunner) Run(ctx context.Context, workspace string, req Request) (Result, error) {
|
||||
func (r *InProcessRunner) Run(ctx context.Context, workspace string, req Request, report ToolCallReporter) (Result, error) {
|
||||
if err := req.Validate(); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
@@ -75,7 +78,9 @@ func (r *InProcessRunner) Run(ctx context.Context, workspace string, req Request
|
||||
|
||||
notes := &memoryNotes{initial: req.Notes}
|
||||
captures := make(map[string][]json.RawMessage)
|
||||
centralTools, err := BuildToolSet(r.tools, r.Limits.Options(workspaceToolset))
|
||||
toolOpts := r.Limits.Options(workspaceToolset)
|
||||
toolOpts.OnToolCall = report
|
||||
centralTools, err := BuildToolSet(r.tools, toolOpts)
|
||||
if err != nil {
|
||||
return Result{}, fmt.Errorf("build central tool set: %w", err)
|
||||
}
|
||||
|
||||
@@ -39,7 +39,7 @@ func TestInProcessRunnerMapsRequestAndCapturesOutput(t *testing.T) {
|
||||
Protection: Protection{Suffixes: []string{"_templ.go"}, Globs: []string{"*_test.go"}},
|
||||
CaptureTools: []CaptureTool{{Name: "decide", Parameters: json.RawMessage(`{"type":"object"}`), MaxCalls: 1}},
|
||||
}
|
||||
got, err := runner.Run(context.Background(), "/workspace", req)
|
||||
got, err := runner.Run(context.Background(), "/workspace", req, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -89,7 +89,7 @@ func TestInProcessRunnerIncludesAvailableCentralTools(t *testing.T) {
|
||||
Stage: "build",
|
||||
Model: "provider/model",
|
||||
Mission: "mission",
|
||||
})
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -121,7 +121,7 @@ func TestInProcessRunnerRejectsInvalidCentralToolSchemas(t *testing.T) {
|
||||
Stage: "build",
|
||||
Model: "provider/model",
|
||||
Mission: "mission",
|
||||
})
|
||||
}, nil)
|
||||
if err == nil {
|
||||
t.Fatal("Run() error = nil, want invalid central tool schema error")
|
||||
}
|
||||
@@ -242,7 +242,7 @@ func TestInProcessRunnerPreservesCallerCancellation(t *testing.T) {
|
||||
}
|
||||
_, err := runner.Run(ctx, "/workspace", Request{
|
||||
Version: ProtocolVersion, TaskID: 1, Attempt: 1, Stage: "build", Model: "provider/model", Mission: "mission",
|
||||
})
|
||||
}, nil)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("Run error = %v, want context.Canceled", err)
|
||||
}
|
||||
|
||||
@@ -67,7 +67,10 @@ type NATSRunner struct {
|
||||
|
||||
// Run publishes a stage execution request to JetStream with an X-Archie-Reply
|
||||
// header, blocks on the sync inbox subscription, and returns the result.
|
||||
func (r *NATSRunner) Run(ctx context.Context, workspace string, req Request) (Result, error) {
|
||||
//
|
||||
// report is ignored: execution happens on whatever remote worker answers the
|
||||
// request, a different process with no way to call back here per tool call.
|
||||
func (r *NATSRunner) Run(ctx context.Context, workspace string, req Request, _ ToolCallReporter) (Result, error) {
|
||||
if err := req.Validate(); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ import (
|
||||
|
||||
type mockRunner struct{}
|
||||
|
||||
func (m *mockRunner) Run(_ context.Context, _ string, req Request) (Result, error) {
|
||||
func (m *mockRunner) Run(_ context.Context, _ string, req Request, _ ToolCallReporter) (Result, error) {
|
||||
return Result{
|
||||
Version: ProtocolVersion,
|
||||
TaskID: req.TaskID,
|
||||
@@ -269,7 +269,7 @@ func TestNATSRunnerRoundTrip(t *testing.T) {
|
||||
Budget: Budget{MaxSteps: 5, MaxTokens: 100},
|
||||
}
|
||||
|
||||
result, err := runner.Run(ctx, "/tmp/test-workspace", req)
|
||||
result, err := runner.Run(ctx, "/tmp/test-workspace", req, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -13,7 +13,7 @@ type captureRunner struct {
|
||||
callback func(Request)
|
||||
}
|
||||
|
||||
func (c *captureRunner) Run(_ context.Context, _ string, req Request) (Result, error) {
|
||||
func (c *captureRunner) Run(_ context.Context, _ string, req Request, _ ToolCallReporter) (Result, error) {
|
||||
c.callback(req)
|
||||
return Result{
|
||||
Version: ProtocolVersion,
|
||||
|
||||
@@ -26,7 +26,9 @@ type SubprocessRunner struct {
|
||||
Providers map[string]Provider
|
||||
}
|
||||
|
||||
func (r *SubprocessRunner) Run(ctx context.Context, workspace string, req Request) (Result, error) {
|
||||
// Run ignores report: the subprocess executes tools in its own process, so
|
||||
// there is nothing here to observe a call as it happens.
|
||||
func (r *SubprocessRunner) Run(ctx context.Context, workspace string, req Request, _ ToolCallReporter) (Result, error) {
|
||||
providerName, _, ok := strings.Cut(req.Model, "/")
|
||||
if !ok || providerName == "" {
|
||||
return Result{}, fmt.Errorf("agent model %q must be provider/model", req.Model)
|
||||
|
||||
@@ -15,7 +15,7 @@ import (
|
||||
|
||||
func TestSubprocessRunnerRoundTrip(t *testing.T) {
|
||||
req := testRequest()
|
||||
result, err := helperRunner("success").Run(context.Background(), "/workspace", req)
|
||||
result, err := helperRunner("success").Run(context.Background(), "/workspace", req, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -26,7 +26,7 @@ func TestSubprocessRunnerRoundTrip(t *testing.T) {
|
||||
|
||||
func TestSubprocessRunnerReturnsStampedWorkerError(t *testing.T) {
|
||||
req := testRequest()
|
||||
result, err := helperRunner("worker-error").Run(context.Background(), "/workspace", req)
|
||||
result, err := helperRunner("worker-error").Run(context.Background(), "/workspace", req, nil)
|
||||
if err == nil || err.Error() != "provider unavailable" {
|
||||
t.Fatalf("error = %v", err)
|
||||
}
|
||||
@@ -38,7 +38,7 @@ func TestSubprocessRunnerReturnsStampedWorkerError(t *testing.T) {
|
||||
func TestSubprocessRunnerRejectsMalformedAndOversizedOutput(t *testing.T) {
|
||||
for _, behavior := range []string{"malformed", "oversized", "multiple", "mismatched"} {
|
||||
t.Run(behavior, func(t *testing.T) {
|
||||
_, err := helperRunner(behavior).Run(context.Background(), "/workspace", testRequest())
|
||||
_, err := helperRunner(behavior).Run(context.Background(), "/workspace", testRequest(), nil)
|
||||
if err == nil {
|
||||
t.Fatal("Run succeeded")
|
||||
}
|
||||
@@ -47,7 +47,7 @@ func TestSubprocessRunnerRejectsMalformedAndOversizedOutput(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerReportsExitDiagnostics(t *testing.T) {
|
||||
_, err := helperRunner("exit").Run(context.Background(), "/workspace", testRequest())
|
||||
_, err := helperRunner("exit").Run(context.Background(), "/workspace", testRequest(), nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "child diagnostic") {
|
||||
t.Fatalf("error = %v", err)
|
||||
}
|
||||
@@ -56,7 +56,7 @@ func TestSubprocessRunnerReportsExitDiagnostics(t *testing.T) {
|
||||
func TestSubprocessRunnerCancelsChild(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
|
||||
defer cancel()
|
||||
_, err := helperRunner("hang").Run(ctx, "/workspace", testRequest())
|
||||
_, err := helperRunner("hang").Run(ctx, "/workspace", testRequest(), nil)
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("error = %v, want deadline exceeded", err)
|
||||
}
|
||||
@@ -69,7 +69,7 @@ func TestSubprocessRunnerCancellationKillsGrandchildren(t *testing.T) {
|
||||
runner.AdditionalEnv = append(runner.AdditionalEnv, "AGENT_HELPER_MARKER")
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
|
||||
defer cancel()
|
||||
_, err := runner.Run(ctx, "/workspace", testRequest())
|
||||
_, err := runner.Run(ctx, "/workspace", testRequest(), nil)
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("error = %v, want deadline exceeded", err)
|
||||
}
|
||||
@@ -83,7 +83,7 @@ func TestSubprocessRunnerForwardsOnlyRequestedProvider(t *testing.T) {
|
||||
runner := helperRunner("provider-scope")
|
||||
runner.Environ = append(runner.Environ, "OTHER_KEY=must-not-leak")
|
||||
runner.Providers["other"] = Provider{Class: "openai", APIKeyEnv: "OTHER_KEY"}
|
||||
if _, err := runner.Run(context.Background(), "/workspace", testRequest()); err != nil {
|
||||
if _, err := runner.Run(context.Background(), "/workspace", testRequest(), nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -137,7 +137,7 @@ func TestServeOneWritesStampedExecutionError(t *testing.T) {
|
||||
}
|
||||
var output strings.Builder
|
||||
err = serveOne(context.Background(), strings.NewReader(string(input)), &output, func(Invocation) Runner {
|
||||
return runnerFunc(func(_ context.Context, _ string, req Request) (Result, error) {
|
||||
return runnerFunc(func(_ context.Context, _ string, req Request, _ ToolCallReporter) (Result, error) {
|
||||
return Result{
|
||||
Version: ProtocolVersion, TaskID: req.TaskID, Attempt: req.Attempt,
|
||||
Stage: req.Stage, Status: StatusPassed,
|
||||
@@ -176,10 +176,10 @@ func helperRunner(behavior string) *SubprocessRunner {
|
||||
}
|
||||
}
|
||||
|
||||
type runnerFunc func(context.Context, string, Request) (Result, error)
|
||||
type runnerFunc func(context.Context, string, Request, ToolCallReporter) (Result, error)
|
||||
|
||||
func (f runnerFunc) Run(ctx context.Context, workspace string, req Request) (Result, error) {
|
||||
return f(ctx, workspace, req)
|
||||
func (f runnerFunc) Run(ctx context.Context, workspace string, req Request, report ToolCallReporter) (Result, error) {
|
||||
return f(ctx, workspace, req, report)
|
||||
}
|
||||
|
||||
func testRequest() Request {
|
||||
|
||||
@@ -83,6 +83,56 @@ type ToolSetOptions struct {
|
||||
// Nil means no approver is configured — such tools are omitted from the
|
||||
// built set rather than failing at call time.
|
||||
Approval tools.ApprovalRequester
|
||||
|
||||
// OnToolCall, when non-nil, is notified once per completed tool call.
|
||||
// InProcessRunner populates it so a workflow stage can surface tool
|
||||
// activity on the task timeline (archie-core-467's task-transcript
|
||||
// counterpart). Nil means no one is listening.
|
||||
OnToolCall ToolCallReporter
|
||||
}
|
||||
|
||||
// ToolCallReport is one completed tool invocation, reported through a
|
||||
// ToolCallReporter.
|
||||
type ToolCallReport struct {
|
||||
Tool string
|
||||
// Detail is a clipped one-line summary: the tool's result, or
|
||||
// "error: <message>" if it failed.
|
||||
Detail string
|
||||
Failed bool
|
||||
}
|
||||
|
||||
// ToolCallReporter receives one ToolCallReport per completed tool call. Only
|
||||
// a runner that executes in the same process as its tools can populate this
|
||||
// -- SubprocessRunner and NATSRunner dispatch to a different process and have
|
||||
// no visibility into individual calls from here. A nil reporter is valid and
|
||||
// reports nothing.
|
||||
type ToolCallReporter func(ToolCallReport)
|
||||
|
||||
// toolCallDetailBytes caps the summary ToolCallReport.Detail carries. This
|
||||
// rides on every task's observability event stream, so one large file read
|
||||
// must not dominate it the way baselineMissionBytes-sized content is allowed
|
||||
// to dominate a builder's own context.
|
||||
const toolCallDetailBytes = 300
|
||||
|
||||
func clipToolCallDetail(s string) string {
|
||||
if len(s) <= toolCallDetailBytes {
|
||||
return s
|
||||
}
|
||||
return s[:toolCallDetailBytes] + "…"
|
||||
}
|
||||
|
||||
// reportToolCallCompletion notifies opts.OnToolCall, if set, of one
|
||||
// completed call. Factored out of toolExecute's closure so the branching
|
||||
// here doesn't add to that function's own cognitive-complexity budget.
|
||||
func reportToolCallCompletion(opts ToolSetOptions, entry tools.ToolEntry, result string, err error) {
|
||||
if opts.OnToolCall == nil {
|
||||
return
|
||||
}
|
||||
outcome := result
|
||||
if err != nil {
|
||||
outcome = "error: " + err.Error()
|
||||
}
|
||||
opts.OnToolCall(ToolCallReport{Tool: entry.Name, Detail: clipToolCallDetail(outcome), Failed: err != nil})
|
||||
}
|
||||
|
||||
// resultLimit returns the cap that applies to one entry.
|
||||
@@ -194,7 +244,9 @@ func safeResolvedSchema(entry tools.ToolEntry) (schema tools.JSONSchema, err err
|
||||
// value would bound the wrong string.
|
||||
func toolExecute(entry tools.ToolEntry, opts ToolSetOptions) func(context.Context, string) (string, error) {
|
||||
limit := opts.resultLimit(entry)
|
||||
return func(ctx context.Context, input string) (string, error) {
|
||||
return func(ctx context.Context, input string) (result string, err error) {
|
||||
defer func() { reportToolCallCompletion(opts, entry, result, err) }()
|
||||
|
||||
// A turn that has spent its budget must say so. Returning an error
|
||||
// tells the model it has run out; returning an empty result would
|
||||
// read as "the tool found nothing" and invite it to try again.
|
||||
|
||||
@@ -87,6 +87,61 @@ func TestToolSetHandlerPropagatesHandlerError(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestToolSetReportsCompletedToolCalls pins archie-core-467's task-transcript
|
||||
// counterpart: every tool invocation built through BuildToolSet must notify
|
||||
// ToolSetOptions.OnToolCall exactly once, on both the success and failure
|
||||
// path, so a workflow stage can surface it on the task timeline.
|
||||
func TestToolSetReportsCompletedToolCalls(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
handler tools.Handler
|
||||
wantFailed bool
|
||||
wantDetail string
|
||||
}{
|
||||
{
|
||||
name: "success",
|
||||
handler: func(context.Context, map[string]any) (any, error) { return "wrote 3 lines", nil },
|
||||
wantFailed: false,
|
||||
wantDetail: `"wrote 3 lines"`,
|
||||
},
|
||||
{
|
||||
name: "failure",
|
||||
handler: func(context.Context, map[string]any) (any, error) { return nil, errors.New("permission denied") },
|
||||
wantFailed: true,
|
||||
wantDetail: "error: permission denied",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
reg := tools.NewRegistry()
|
||||
if err := reg.Register(tools.ToolEntry{Name: "write_file", Handler: tt.handler}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var reports []ToolCallReport
|
||||
set := mustBuildToolSetWith(t, reg, ToolSetOptions{
|
||||
OnToolCall: func(rep ToolCallReport) { reports = append(reports, rep) },
|
||||
})
|
||||
_, _ = set["write_file"].Execute(context.Background(), `{}`)
|
||||
|
||||
if len(reports) != 1 {
|
||||
t.Fatalf("OnToolCall was called %d times, want 1: %+v", len(reports), reports)
|
||||
}
|
||||
got := reports[0]
|
||||
if got.Tool != "write_file" {
|
||||
t.Errorf("Tool = %q, want %q", got.Tool, "write_file")
|
||||
}
|
||||
if got.Failed != tt.wantFailed {
|
||||
t.Errorf("Failed = %v, want %v", got.Failed, tt.wantFailed)
|
||||
}
|
||||
if got.Detail != tt.wantDetail {
|
||||
t.Errorf("Detail = %q, want %q", got.Detail, tt.wantDetail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestToolSetHandlerRejectsMalformedInput(t *testing.T) {
|
||||
reg := tools.NewRegistry()
|
||||
if err := reg.Register(tools.ToolEntry{
|
||||
|
||||
@@ -36,7 +36,7 @@ func serveOne(ctx context.Context, in io.Reader, out io.Writer, newRunner func(I
|
||||
if runner == nil {
|
||||
return fmt.Errorf("agent runner is not configured")
|
||||
}
|
||||
result, runErr := runner.Run(ctx, invocation.Workspace, invocation.Request)
|
||||
result, runErr := runner.Run(ctx, invocation.Workspace, invocation.Request, nil)
|
||||
response := Response{Version: ProtocolVersion, Result: result}
|
||||
if runErr != nil {
|
||||
response.Error = runErr.Error()
|
||||
@@ -57,6 +57,12 @@ type RunnerFactory func(Providers map[string]Provider, log *slog.Logger) Runner
|
||||
// HandleMessage processes one AgentRequestMessage. When msg.Stages is
|
||||
// populated and msg.Workflow is set, it runs all stages as a per-task
|
||||
// batch. Otherwise it falls back to single-stage execution.
|
||||
//
|
||||
// It passes a nil ToolCallReporter to every Run call: this path has no
|
||||
// TaskContext and therefore no events.Bus to report through. The workflow
|
||||
// engine's own AgentStage.Stage().Run (agent.go) is what wires a real
|
||||
// reporter into tc.Emit; this entry point predates that and is not reached
|
||||
// by workflow.Run.
|
||||
func HandleMessage(ctx context.Context, msg AgentRequestMessage, log *slog.Logger, newRunner RunnerFactory) (*AgentResponseEnvelope, error) {
|
||||
runner := newRunner(msg.Providers, log)
|
||||
if runner == nil {
|
||||
@@ -74,7 +80,7 @@ func HandleMessage(ctx context.Context, msg AgentRequestMessage, log *slog.Logge
|
||||
}
|
||||
|
||||
// Single-stage mode (backward compatible).
|
||||
result, runErr := runner.Run(ctx, msg.Workspace, msg.Request)
|
||||
result, runErr := runner.Run(ctx, msg.Workspace, msg.Request, nil)
|
||||
resp := &AgentResponseEnvelope{
|
||||
Version: msg.Request.Version,
|
||||
Result: result,
|
||||
@@ -100,7 +106,7 @@ func runStages(ctx context.Context, runner Runner, msg AgentRequestMessage, chan
|
||||
|
||||
for i, stage := range msg.Stages {
|
||||
log.Info("running stage", "stage", stage.Stage, "batch", fmt.Sprintf("%d/%d", i+1, len(msg.Stages)))
|
||||
result, runErr := runner.Run(ctx, msg.Workspace, stage)
|
||||
result, runErr := runner.Run(ctx, msg.Workspace, stage, nil)
|
||||
if runErr != nil {
|
||||
resp := &AgentResponseEnvelope{
|
||||
Version: stage.Version,
|
||||
|
||||
@@ -30,7 +30,7 @@ import (
|
||||
|
||||
type panicRunner struct{ t *testing.T }
|
||||
|
||||
func (r panicRunner) Run(context.Context, string, agentexec.Request) (agentexec.Result, error) {
|
||||
func (r panicRunner) Run(context.Context, string, agentexec.Request, agentexec.ToolCallReporter) (agentexec.Result, error) {
|
||||
r.t.Fatal("bootstrap workflow must not invoke an LLM runner")
|
||||
return agentexec.Result{}, nil
|
||||
}
|
||||
|
||||
@@ -34,7 +34,7 @@ type cleanupStorage struct {
|
||||
|
||||
type identityRunnerStub struct{}
|
||||
|
||||
func (*identityRunnerStub) Run(context.Context, string, agentexec.Request) (agentexec.Result, error) {
|
||||
func (*identityRunnerStub) Run(context.Context, string, agentexec.Request, agentexec.ToolCallReporter) (agentexec.Result, error) {
|
||||
return agentexec.Result{}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -138,7 +138,7 @@ func (a AgentStage) Stage() Stage {
|
||||
CaptureTools: captureTools,
|
||||
Plugins: pluginSpecs(tc.SkillPlugins),
|
||||
}
|
||||
res, err := tc.Agent.Run(ctx, tc.Dir, req)
|
||||
res, err := tc.Agent.Run(ctx, tc.Dir, req, tc.toolCallReporter(a.Name))
|
||||
|
||||
// Record guardrail state: on success the guardrail engine checks
|
||||
// no-progress thresholds; on failure it records the error pattern.
|
||||
|
||||
@@ -9,9 +9,11 @@ import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/agentexec"
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/events"
|
||||
"github.com/samcharles93/archie-core/internal/store"
|
||||
)
|
||||
|
||||
@@ -20,17 +22,25 @@ type fakeAgentRunner struct {
|
||||
request agentexec.Request
|
||||
result agentexec.Result
|
||||
err error
|
||||
// report is the reporter the caller passed to Run, captured so a test
|
||||
// can invoke it directly and assert on the resulting tc.Emit call.
|
||||
report agentexec.ToolCallReporter
|
||||
}
|
||||
|
||||
type agentRunnerFunc func(context.Context, string, agentexec.Request) (agentexec.Result, error)
|
||||
type agentRunnerFunc func(context.Context, string, agentexec.Request, agentexec.ToolCallReporter) (agentexec.Result, error)
|
||||
|
||||
func (f agentRunnerFunc) Run(ctx context.Context, workspace string, req agentexec.Request) (agentexec.Result, error) {
|
||||
return f(ctx, workspace, req)
|
||||
func (f agentRunnerFunc) Run(
|
||||
ctx context.Context, workspace string, req agentexec.Request, report agentexec.ToolCallReporter,
|
||||
) (agentexec.Result, error) {
|
||||
return f(ctx, workspace, req, report)
|
||||
}
|
||||
|
||||
func (r *fakeAgentRunner) Run(_ context.Context, workspace string, req agentexec.Request) (agentexec.Result, error) {
|
||||
func (r *fakeAgentRunner) Run(
|
||||
_ context.Context, workspace string, req agentexec.Request, report agentexec.ToolCallReporter,
|
||||
) (agentexec.Result, error) {
|
||||
r.workspace = workspace
|
||||
r.request = req
|
||||
r.report = report
|
||||
if r.result.Version == 0 {
|
||||
r.result.Version = agentexec.ProtocolVersion
|
||||
}
|
||||
@@ -83,6 +93,61 @@ func TestAgentStageBuildsExecutionRequestAndAppliesResult(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestAgentStageReportsToolCallsOnTheTaskBus pins archie-core-467's
|
||||
// task-transcript counterpart: the reporter AgentStage.Stage() passes into
|
||||
// tc.Agent.Run must, when invoked, publish a tool_call event onto tc.Bus
|
||||
// carrying the tool name and outcome -- otherwise a workflow task's own
|
||||
// transcript never shows what its tools actually did, only the chat
|
||||
// gateway's interactive turns do.
|
||||
func TestAgentStageReportsToolCallsOnTheTaskBus(t *testing.T) {
|
||||
runner := &fakeAgentRunner{result: agentexec.Result{
|
||||
Version: agentexec.ProtocolVersion, Status: agentexec.StatusPassed,
|
||||
}}
|
||||
stage := AgentStage{
|
||||
Name: "build",
|
||||
Role: "builder",
|
||||
Mission: func(*TaskContext) string { return "implement the feature" },
|
||||
}.Stage()
|
||||
|
||||
bus := events.NewBus()
|
||||
sub := bus.Subscribe(8)
|
||||
t.Cleanup(sub.Close)
|
||||
|
||||
task := &store.Task{ID: 9, Owner: "acme", Repo: "widget", Attempt: 1}
|
||||
tc := &TaskContext{
|
||||
Task: task, Agent: runner, Dir: "/tmp/workspace", Bus: bus, Log: slog.New(slog.DiscardHandler),
|
||||
Cfg: config.Config{Models: map[string]string{"builder": "provider/model"}},
|
||||
}
|
||||
if err := stage.Run(context.Background(), tc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if runner.report == nil {
|
||||
t.Fatal("AgentStage did not pass a ToolCallReporter to Run")
|
||||
}
|
||||
runner.report(agentexec.ToolCallReport{Tool: "read_file", Detail: `"package main"`, Failed: false})
|
||||
runner.report(agentexec.ToolCallReport{Tool: "shell", Detail: "error: exit status 1", Failed: true})
|
||||
|
||||
var got []events.Event
|
||||
for len(got) < 2 {
|
||||
select {
|
||||
case e := <-sub.C:
|
||||
if e.Kind == events.KindToolCall {
|
||||
got = append(got, e)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatalf("timed out waiting for tool_call events, got %d", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
if got[0].Stage != "build" || got[0].Data["tool"] != "read_file" || got[0].Detail != `"package main"` || got[0].Data["failed"] != false {
|
||||
t.Fatalf("first tool_call event = %#v", got[0])
|
||||
}
|
||||
if got[1].Data["tool"] != "shell" || got[1].Detail != "error: exit status 1" || got[1].Data["failed"] != true {
|
||||
t.Fatalf("second tool_call event = %#v", got[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentStagePersistsReturnedNotes(t *testing.T) {
|
||||
st, err := store.Open(t.Context(), filepath.Join(t.TempDir(), "archie.db"))
|
||||
if err != nil {
|
||||
@@ -122,7 +187,7 @@ func TestAgentStagePersistsReturnedNotes(t *testing.T) {
|
||||
|
||||
func TestAgentStageDiscardsUnstampedErrorResult(t *testing.T) {
|
||||
task := &store.Task{ID: 1, Attempt: 1, Notes: "existing\n"}
|
||||
runner := agentRunnerFunc(func(context.Context, string, agentexec.Request) (agentexec.Result, error) {
|
||||
runner := agentRunnerFunc(func(context.Context, string, agentexec.Request, agentexec.ToolCallReporter) (agentexec.Result, error) {
|
||||
return agentexec.Result{AppendedNotes: []string{"untrusted"}}, errors.New("worker exited")
|
||||
})
|
||||
stage := AgentStage{
|
||||
|
||||
@@ -81,7 +81,7 @@ func StageBaselineGate() Stage {
|
||||
Protection: agentexec.Protection{Suffixes: append([]string(nil), tc.Repo.Protect...)},
|
||||
}
|
||||
|
||||
res, agentErr := tc.Agent.Run(ctx, tc.Dir, req)
|
||||
res, agentErr := tc.Agent.Run(ctx, tc.Dir, req, tc.toolCallReporter("baseline-fix"))
|
||||
if agentErr != nil {
|
||||
return fmt.Errorf("baseline-fix agent run: %w", agentErr)
|
||||
}
|
||||
|
||||
@@ -163,7 +163,7 @@ func TestStageCommitPushDoesNotUseSyntheticIssueForChatNoOp(t *testing.T) {
|
||||
|
||||
// alwaysFailingRunner reports the gate as unfixable, so StageBaselineGate
|
||||
// falls through to its terminal error without needing a real agent loop.
|
||||
var alwaysFailingRunner = agentRunnerFunc(func(context.Context, string, agentexec.Request) (agentexec.Result, error) {
|
||||
var alwaysFailingRunner = agentRunnerFunc(func(context.Context, string, agentexec.Request, agentexec.ToolCallReporter) (agentexec.Result, error) {
|
||||
return agentexec.Result{Status: "failed"}, nil
|
||||
})
|
||||
|
||||
|
||||
@@ -119,6 +119,20 @@ func (tc *TaskContext) Emit(kind, stage, detail string, data map[string]any) {
|
||||
})
|
||||
}
|
||||
|
||||
// toolCallReporter builds the agentexec.ToolCallReporter an agent stage
|
||||
// passes to tc.Agent.Run, so every completed tool call during that run
|
||||
// surfaces as a tool_call event on the task timeline (archie-core-467's
|
||||
// task-transcript counterpart -- the interactive chat gateway has its own,
|
||||
// separate ToolCallEvent). Safe to call on a nil Bus: Emit no-ops.
|
||||
func (tc *TaskContext) toolCallReporter(stage string) agentexec.ToolCallReporter {
|
||||
return func(report agentexec.ToolCallReport) {
|
||||
tc.Emit(events.KindToolCall, stage, report.Detail, map[string]any{
|
||||
"tool": report.Tool,
|
||||
"failed": report.Failed,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Outcome is a stage's terminal decision for the whole workflow. Stages
|
||||
// that don't end the workflow leave it zero.
|
||||
type Outcome struct {
|
||||
|
||||
@@ -16,11 +16,17 @@ const (
|
||||
KindStageStart = "stage_start"
|
||||
KindStageFinish = "stage_finish" // data: duration_ms, error
|
||||
KindAgentFinish = "agent_finish" // data: status, stop_reason, tokens, iterations, model
|
||||
KindParked = "parked" // data: reason
|
||||
KindOutcome = "outcome" // data: status, detail
|
||||
KindPRMerged = "pr_merged"
|
||||
KindPRRejected = "pr_rejected"
|
||||
KindTaskDead = "task_dead"
|
||||
// KindToolCall marks one completed tool invocation during an agent
|
||||
// stage run (archie-core-467's task-transcript counterpart -- the
|
||||
// interactive chat gateway surfaces its own tool calls separately via
|
||||
// gateway.ToolCallEvent). Only emitted for runners that execute in the
|
||||
// same process as their tools; see agentexec.ToolCallReporter.
|
||||
KindToolCall = "tool_call" // data: tool, failed
|
||||
KindParked = "parked" // data: reason
|
||||
KindOutcome = "outcome" // data: status, detail
|
||||
KindPRMerged = "pr_merged"
|
||||
KindPRRejected = "pr_rejected"
|
||||
KindTaskDead = "task_dead"
|
||||
// Operator actions taken from the dashboard. Without these an
|
||||
// intervention is recorded in the tasks table but invisible on the
|
||||
// timeline and the activity stream -- the one kind of event most worth
|
||||
|
||||
Reference in new issue
Block a user