Files
archie/modules/runs/agent_test.go
T
samandCopilot 64dfad6d53 feat(runs): add typed results and model aliases (#2043)
Also fix repository-free message.send validation (#2046) and add a single-binary development smoke path (#1977).

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-11-18 23:47:35 +11:00

240 lines
8.8 KiB
Go

package runs
import (
"context"
"encoding/json/jsontext"
json "encoding/json/v2"
"errors"
"log/slog"
"os"
"slices"
"strings"
"sync"
"testing"
"github.com/google/uuid"
"github.com/samcharles93/ai-sdk/chat"
"github.com/samcharles93/ai-sdk/core"
"github.com/samcharles93/archie-core/modules/governance"
"github.com/samcharles93/archie-core/modules/runs/internal/store"
)
type triageModel struct {
scriptedModel
models []string
}
func (m *triageModel) Chat(_ context.Context, req chat.Request) (chat.Response, error) {
m.models = append(m.models, req.Model)
if req.Messages[len(req.Messages)-1].Role == chat.RoleTool {
return chat.Response{Role: chat.RoleAssistant, Content: "No change needed.", FinishReason: "stop"}, nil
}
return chat.Response{
Role: chat.RoleAssistant, FinishReason: "tool_calls",
ToolCalls: []chat.ToolCall{{ID: "classification", Name: "return_result", Arguments: `{"needs_code_change":false,"reasons":"Already resolved"}`}},
}, nil
}
type modelSettings struct {
onePersonPath
model *triageModel
name string
}
func (s modelSettings) AgentModelForAlias(_ context.Context, alias string) (governance.Model, error) {
if alias != "fast" {
return governance.Model{}, governance.ErrNoAgentModel
}
return governance.Model{Model: s.name}, nil
}
func (s modelSettings) AgentProvider(context.Context, governance.Model) (chat.Provider, chat.ModelInfo, error) {
return s.model, chat.ModelInfo{}, nil
}
func (modelSettings) AttemptsPerRun(context.Context) (int, error) { return 6, nil }
// TestTriageClassificationUsesPinnedModel checks that a resumed step returns
// the same typed classification even if its model alias now resolves differently.
func TestTriageClassificationUsesPinnedModel(t *testing.T) {
raw, err := os.ReadFile("testdata/corpus/triage.yaml")
if err != nil {
t.Fatal(err)
}
body, _, ok := strings.Cut(string(raw), " - type: forge.comment")
if !ok {
t.Fatal("triage fixture has no forge comment step")
}
body = strings.Replace(body, " read_only: true", " read_only: true\n model: fast", 1)
doc, err := ParseDocument(body)
if err != nil {
t.Fatal(err)
}
engine := newEngine(nil, &fakeWork{}, slog.New(slog.DiscardHandler), Options{})
engine.Use(Deps{Settings: modelSettings{}})
_, err = engine.Install(t.Context(), "org", "triage", strings.Replace(body, "model: fast", "model: missing", 1), "operator")
var invalid *InvalidError
if !errors.As(err, &invalid) || len(invalid.Problems) != 1 || invalid.Problems[0].Path != "steps[0].settings.model" {
t.Fatalf("saving an unknown alias returned %v, want a model setting error", err)
}
run := store.Run{ID: uuid.New()}
work := &fakeWork{}
aliases, withMemory := agentUse(doc)
for _, tt := range []struct {
name string
want []string
}{
{name: "fast-v1", want: []string{"fast-v1", "fast-v1"}},
{name: "fast-v2"},
} {
model := &triageModel{}
deps := Deps{Settings: modelSettings{model: model, name: tt.name}, Toolbox: anyBox{}, Judge: &actionJudge{}}
rc := newContext(t.Context(), work, run, uuid.New(), work.steps)
p := &progress{results: make(map[string]StepResult)}
if err := deps.pin(rc, p, aliases, withMemory); err != nil {
t.Fatal(err)
}
if err := deps.runStep(rc, p, doc.Steps[0], 0); err != nil {
t.Fatal(err)
}
got := p.results["classify"].Result
if got["needs_code_change"] != false || got["reasons"] != "Already resolved" {
t.Fatalf("classification = %#v", got)
}
if !slices.Equal(model.models, tt.want) {
t.Fatalf("model calls = %v, want %v", model.models, tt.want)
}
}
}
// scriptedModel answers by turn: a tool call first, then findings once the
// tool's result is in the history.
type scriptedModel struct {
mu sync.Mutex
calls int
}
func (m *scriptedModel) Name() string { return "scripted" }
func (m *scriptedModel) Chat(_ context.Context, req chat.Request) (chat.Response, error) {
m.mu.Lock()
m.calls++
m.mu.Unlock()
last := req.Messages[len(req.Messages)-1]
if last.Role == chat.RoleTool {
return chat.Response{
Role: chat.RoleAssistant, Content: "disk is full: " + last.Content, FinishReason: "stop",
Parts: chat.Parts{chat.ReasoningPart{Text: "the tool said why"}, chat.TextPart{Text: "disk is full: " + last.Content}},
Usage: chat.Usage{PromptTokens: 20, CompletionTokens: 5, TotalTokens: 25},
}, nil
}
return chat.Response{
Role: chat.RoleAssistant, FinishReason: "tool_calls",
Parts: chat.Parts{chat.ReasoningPart{Text: "look at the disk"}},
ToolCalls: []chat.ToolCall{{ID: "call-1", Name: "disk_usage", Arguments: `{"path":"/tmp"}`}},
Usage: chat.Usage{PromptTokens: 10, CompletionTokens: 3, TotalTokens: 13},
}, nil
}
func (m *scriptedModel) ChatStream(context.Context, chat.Request) (chat.Stream, error) {
return nil, errors.New("not streamed")
}
// TestAgentResumesFromItsRecordedCalls guards the agent loop's checkpoint
// contract (2.2, 2.3): a worker that stops after any recorded call, killed or
// shut down mid-tool, resumes the loop from the run's steps, replays what was
// recorded without calling the model or running the tool again, and ends with
// the same findings. A step encoding that changes a recorded call shows up as
// a replay that diverges from its journal.
func TestAgentResumesFromItsRecordedCalls(t *testing.T) {
// The uninterrupted run records a step before the loop (seq 0), the
// loop's model, tool and model calls (1 to 3), and a step after it (4).
type stop struct {
name string
// killAt makes writes fail as a lost lease from that many steps on.
killAt *int
// shutdownInTool ends the run's context while the tool runs.
shutdownInTool bool
// replayedModel and replayedTool are the calls the resume replays.
replayedModel, replayedTool int
}
at := func(n int) *int { return &n }
tests := []stop{
{name: "killed before the loop", killAt: at(0)},
{name: "killed before the first model call is recorded", killAt: at(1)},
{name: "killed before the tool is recorded", killAt: at(2), replayedModel: 1},
{name: "killed before the second model call is recorded", killAt: at(3), replayedModel: 1, replayedTool: 1},
{name: "killed before the last step", killAt: at(4), replayedModel: 2, replayedTool: 1},
{name: "killed before finishing", killAt: at(5), replayedModel: 2, replayedTool: 1},
{name: "shut down while the tool runs", shutdownInTool: true, replayedModel: 1, replayedTool: 1},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ctx, shutdown := context.WithCancel(t.Context())
defer shutdown()
duringTool := func() {}
if tt.shutdownInTool {
duringTool = shutdown
}
first := &fakeWork{rejectFrom: tt.killAt}
execute(ctx, t, first, &scriptedModel{}, new(int), duringTool)
resumed := &fakeWork{steps: first.steps}
model, toolRuns := &scriptedModel{}, new(int)
execute(t.Context(), t, resumed, model, toolRuns, func() {})
if len(resumed.finished) != 1 || resumed.finished[0] != store.StateSucceeded {
t.Fatalf("resumed run finished %v, want succeeded", resumed.finished)
}
var res AgentResult
if err := json.Unmarshal(resumed.results[0], &res); err != nil {
t.Fatal(err)
}
if res.Answer != "disk is full: 93%" || res.Usage.TotalTokens != 38 {
t.Fatalf("resumed findings = %+v", res)
}
if want := 1 - tt.replayedTool; *toolRuns != want {
t.Fatalf("tool ran %d times on resume, want %d", *toolRuns, want)
}
if want := 2 - tt.replayedModel; model.calls != want {
t.Fatalf("model called %d times on resume, want %d", model.calls, want)
}
})
}
}
// execute runs the probe workflow once on work. duringTool runs inside the
// tool call.
func execute(ctx context.Context, t *testing.T, work *fakeWork, model chat.Provider, toolRuns *int, duringTool func()) {
t.Helper()
engine := newEngine(nil, work, slog.New(slog.DiscardHandler), Options{})
useWorkflow(engine, func(rc *Context, _ jsontext.Value) (jsontext.Value, error) {
if _, err := rc.Step("before", func(context.Context, string) (bool, error) { return true, nil }); err != nil {
return nil, err
}
res, err := rc.Agent(AgentRequest{
Name: "probe", Provider: model, Model: "m", Prompt: "why is the disk full?",
Tools: core.ToolSet{"disk_usage": {
Name: "disk_usage", Parameters: []byte(`{"type":"object"}`),
Execute: func(ctx context.Context, input string) (string, error) {
if !strings.Contains(core.IdempotencyKey(ctx), ":probe:") {
return "", errors.New("no idempotency key")
}
*toolRuns++
duringTool()
return "93%", nil
},
}},
})
if err != nil {
return nil, err
}
return rc.Step("after", func(context.Context, string) (jsontext.Value, error) { return json.Marshal(res) })
})
run := store.Run{ID: runID, OrgID: "org", Workflow: "agent", Version: 1, Input: jsontext.Value(`null`)}
engine.execute(ctx, run, uuid.New())
}
var runID = uuid.MustParse("6b1f6c2e-8d5a-4c43-9f0e-1d2a3b4c5d6e")