Files
archie/modules/runs/agent_test.go
T

149 lines
5.6 KiB
Go

package runs
import (
"context"
"encoding/json/jsontext"
json "encoding/json/v2"
"errors"
"log/slog"
"strings"
"sync"
"testing"
"github.com/google/uuid"
"github.com/samcharles93/ai-sdk/chat"
"github.com/samcharles93/ai-sdk/core"
"github.com/samcharles93/archie-core/modules/runs/internal/store"
)
// scriptedModel answers by turn: a tool call first, then findings once the
// tool's result is in the history.
type scriptedModel struct {
mu sync.Mutex
calls int
}
func (m *scriptedModel) Name() string { return "scripted" }
func (m *scriptedModel) Chat(_ context.Context, req chat.Request) (chat.Response, error) {
m.mu.Lock()
m.calls++
m.mu.Unlock()
last := req.Messages[len(req.Messages)-1]
if last.Role == chat.RoleTool {
return chat.Response{
Role: chat.RoleAssistant, Content: "disk is full: " + last.Content, FinishReason: "stop",
Parts: chat.Parts{chat.ReasoningPart{Text: "the tool said why"}, chat.TextPart{Text: "disk is full: " + last.Content}},
Usage: chat.Usage{PromptTokens: 20, CompletionTokens: 5, TotalTokens: 25},
}, nil
}
return chat.Response{
Role: chat.RoleAssistant, FinishReason: "tool_calls",
Parts: chat.Parts{chat.ReasoningPart{Text: "look at the disk"}},
ToolCalls: []chat.ToolCall{{ID: "call-1", Name: "disk_usage", Arguments: `{"path":"/tmp"}`}},
Usage: chat.Usage{PromptTokens: 10, CompletionTokens: 3, TotalTokens: 13},
}, nil
}
func (m *scriptedModel) ChatStream(context.Context, chat.Request) (chat.Stream, error) {
return nil, errors.New("not streamed")
}
// TestAgentResumesFromItsRecordedCalls guards the agent loop's checkpoint
// contract (2.2, 2.3): a worker that stops after any recorded call, killed or
// shut down mid-tool, resumes the loop from the run's steps, replays what was
// recorded without calling the model or running the tool again, and ends with
// the same findings. A step encoding that changes a recorded call shows up as
// a replay that diverges from its journal.
func TestAgentResumesFromItsRecordedCalls(t *testing.T) {
// The uninterrupted run records a step before the loop (seq 0), the
// loop's model, tool and model calls (1 to 3), and a step after it (4).
type stop struct {
name string
// killAt makes writes fail as a lost lease from that many steps on.
killAt *int
// shutdownInTool ends the run's context while the tool runs.
shutdownInTool bool
// replayedModel and replayedTool are the calls the resume replays.
replayedModel, replayedTool int
}
at := func(n int) *int { return &n }
tests := []stop{
{name: "killed before the loop", killAt: at(0)},
{name: "killed before the first model call is recorded", killAt: at(1)},
{name: "killed before the tool is recorded", killAt: at(2), replayedModel: 1},
{name: "killed before the second model call is recorded", killAt: at(3), replayedModel: 1, replayedTool: 1},
{name: "killed before the last step", killAt: at(4), replayedModel: 2, replayedTool: 1},
{name: "killed before finishing", killAt: at(5), replayedModel: 2, replayedTool: 1},
{name: "shut down while the tool runs", shutdownInTool: true, replayedModel: 1, replayedTool: 1},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ctx, shutdown := context.WithCancel(t.Context())
defer shutdown()
duringTool := func() {}
if tt.shutdownInTool {
duringTool = shutdown
}
first := &fakeWork{rejectFrom: tt.killAt}
execute(ctx, t, first, &scriptedModel{}, new(int), duringTool)
resumed := &fakeWork{steps: first.steps}
model, toolRuns := &scriptedModel{}, new(int)
execute(t.Context(), t, resumed, model, toolRuns, func() {})
if len(resumed.finished) != 1 || resumed.finished[0] != store.StateSucceeded {
t.Fatalf("resumed run finished %v, want succeeded", resumed.finished)
}
var res AgentResult
if err := json.Unmarshal(resumed.results[0], &res); err != nil {
t.Fatal(err)
}
if res.Answer != "disk is full: 93%" || res.Usage.TotalTokens != 38 {
t.Fatalf("resumed findings = %+v", res)
}
if want := 1 - tt.replayedTool; *toolRuns != want {
t.Fatalf("tool ran %d times on resume, want %d", *toolRuns, want)
}
if want := 2 - tt.replayedModel; model.calls != want {
t.Fatalf("model called %d times on resume, want %d", model.calls, want)
}
})
}
}
// execute runs the probe workflow once on work. duringTool runs inside the
// tool call.
func execute(ctx context.Context, t *testing.T, work *fakeWork, model chat.Provider, toolRuns *int, duringTool func()) {
t.Helper()
engine := newEngine(nil, work, slog.New(slog.DiscardHandler), Options{})
useWorkflow(engine, func(rc *Context, _ jsontext.Value) (jsontext.Value, error) {
if _, err := rc.Step("before", func(context.Context, string) (bool, error) { return true, nil }); err != nil {
return nil, err
}
res, err := rc.Agent(AgentRequest{
Name: "probe", Provider: model, Model: "m", Prompt: "why is the disk full?",
Tools: core.ToolSet{"disk_usage": {
Name: "disk_usage", Parameters: []byte(`{"type":"object"}`),
Execute: func(ctx context.Context, input string) (string, error) {
if !strings.Contains(core.IdempotencyKey(ctx), ":probe:") {
return "", errors.New("no idempotency key")
}
*toolRuns++
duringTool()
return "93%", nil
},
}},
})
if err != nil {
return nil, err
}
return rc.Step("after", func(context.Context, string) (jsontext.Value, error) { return json.Marshal(res) })
})
run := store.Run{ID: runID, OrgID: "org", Workflow: "agent", Version: 1, Input: jsontext.Value(`null`)}
engine.execute(ctx, run, uuid.New())
}
var runID = uuid.MustParse("6b1f6c2e-8d5a-4c43-9f0e-1d2a3b4c5d6e")