149 lines
5.6 KiB
Go
149 lines
5.6 KiB
Go
package runs
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json/jsontext"
|
|
json "encoding/json/v2"
|
|
"errors"
|
|
"log/slog"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/samcharles93/ai-sdk/chat"
|
|
"github.com/samcharles93/ai-sdk/core"
|
|
|
|
"github.com/samcharles93/archie-core/modules/runs/internal/store"
|
|
)
|
|
|
|
// scriptedModel answers by turn: a tool call first, then findings once the
|
|
// tool's result is in the history.
|
|
type scriptedModel struct {
|
|
mu sync.Mutex
|
|
calls int
|
|
}
|
|
|
|
func (m *scriptedModel) Name() string { return "scripted" }
|
|
|
|
func (m *scriptedModel) Chat(_ context.Context, req chat.Request) (chat.Response, error) {
|
|
m.mu.Lock()
|
|
m.calls++
|
|
m.mu.Unlock()
|
|
last := req.Messages[len(req.Messages)-1]
|
|
if last.Role == chat.RoleTool {
|
|
return chat.Response{
|
|
Role: chat.RoleAssistant, Content: "disk is full: " + last.Content, FinishReason: "stop",
|
|
Parts: chat.Parts{chat.ReasoningPart{Text: "the tool said why"}, chat.TextPart{Text: "disk is full: " + last.Content}},
|
|
Usage: chat.Usage{PromptTokens: 20, CompletionTokens: 5, TotalTokens: 25},
|
|
}, nil
|
|
}
|
|
return chat.Response{
|
|
Role: chat.RoleAssistant, FinishReason: "tool_calls",
|
|
Parts: chat.Parts{chat.ReasoningPart{Text: "look at the disk"}},
|
|
ToolCalls: []chat.ToolCall{{ID: "call-1", Name: "disk_usage", Arguments: `{"path":"/tmp"}`}},
|
|
Usage: chat.Usage{PromptTokens: 10, CompletionTokens: 3, TotalTokens: 13},
|
|
}, nil
|
|
}
|
|
|
|
func (m *scriptedModel) ChatStream(context.Context, chat.Request) (chat.Stream, error) {
|
|
return nil, errors.New("not streamed")
|
|
}
|
|
|
|
// TestAgentResumesFromItsRecordedCalls guards the agent loop's checkpoint
|
|
// contract (2.2, 2.3): a worker that stops after any recorded call, killed or
|
|
// shut down mid-tool, resumes the loop from the run's steps, replays what was
|
|
// recorded without calling the model or running the tool again, and ends with
|
|
// the same findings. A step encoding that changes a recorded call shows up as
|
|
// a replay that diverges from its journal.
|
|
func TestAgentResumesFromItsRecordedCalls(t *testing.T) {
|
|
// The uninterrupted run records a step before the loop (seq 0), the
|
|
// loop's model, tool and model calls (1 to 3), and a step after it (4).
|
|
type stop struct {
|
|
name string
|
|
// killAt makes writes fail as a lost lease from that many steps on.
|
|
killAt *int
|
|
// shutdownInTool ends the run's context while the tool runs.
|
|
shutdownInTool bool
|
|
// replayedModel and replayedTool are the calls the resume replays.
|
|
replayedModel, replayedTool int
|
|
}
|
|
at := func(n int) *int { return &n }
|
|
tests := []stop{
|
|
{name: "killed before the loop", killAt: at(0)},
|
|
{name: "killed before the first model call is recorded", killAt: at(1)},
|
|
{name: "killed before the tool is recorded", killAt: at(2), replayedModel: 1},
|
|
{name: "killed before the second model call is recorded", killAt: at(3), replayedModel: 1, replayedTool: 1},
|
|
{name: "killed before the last step", killAt: at(4), replayedModel: 2, replayedTool: 1},
|
|
{name: "killed before finishing", killAt: at(5), replayedModel: 2, replayedTool: 1},
|
|
{name: "shut down while the tool runs", shutdownInTool: true, replayedModel: 1, replayedTool: 1},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
ctx, shutdown := context.WithCancel(t.Context())
|
|
defer shutdown()
|
|
duringTool := func() {}
|
|
if tt.shutdownInTool {
|
|
duringTool = shutdown
|
|
}
|
|
first := &fakeWork{rejectFrom: tt.killAt}
|
|
execute(ctx, t, first, &scriptedModel{}, new(int), duringTool)
|
|
|
|
resumed := &fakeWork{steps: first.steps}
|
|
model, toolRuns := &scriptedModel{}, new(int)
|
|
execute(t.Context(), t, resumed, model, toolRuns, func() {})
|
|
|
|
if len(resumed.finished) != 1 || resumed.finished[0] != store.StateSucceeded {
|
|
t.Fatalf("resumed run finished %v, want succeeded", resumed.finished)
|
|
}
|
|
var res AgentResult
|
|
if err := json.Unmarshal(resumed.results[0], &res); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if res.Answer != "disk is full: 93%" || res.Usage.TotalTokens != 38 {
|
|
t.Fatalf("resumed findings = %+v", res)
|
|
}
|
|
if want := 1 - tt.replayedTool; *toolRuns != want {
|
|
t.Fatalf("tool ran %d times on resume, want %d", *toolRuns, want)
|
|
}
|
|
if want := 2 - tt.replayedModel; model.calls != want {
|
|
t.Fatalf("model called %d times on resume, want %d", model.calls, want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// execute runs the probe workflow once on work. duringTool runs inside the
|
|
// tool call.
|
|
func execute(ctx context.Context, t *testing.T, work *fakeWork, model chat.Provider, toolRuns *int, duringTool func()) {
|
|
t.Helper()
|
|
engine := newEngine(nil, work, slog.New(slog.DiscardHandler), Options{})
|
|
useWorkflow(engine, func(rc *Context, _ jsontext.Value) (jsontext.Value, error) {
|
|
if _, err := rc.Step("before", func(context.Context, string) (bool, error) { return true, nil }); err != nil {
|
|
return nil, err
|
|
}
|
|
res, err := rc.Agent(AgentRequest{
|
|
Name: "probe", Provider: model, Model: "m", Prompt: "why is the disk full?",
|
|
Tools: core.ToolSet{"disk_usage": {
|
|
Name: "disk_usage", Parameters: []byte(`{"type":"object"}`),
|
|
Execute: func(ctx context.Context, input string) (string, error) {
|
|
if !strings.Contains(core.IdempotencyKey(ctx), ":probe:") {
|
|
return "", errors.New("no idempotency key")
|
|
}
|
|
*toolRuns++
|
|
duringTool()
|
|
return "93%", nil
|
|
},
|
|
}},
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return rc.Step("after", func(context.Context, string) (jsontext.Value, error) { return json.Marshal(res) })
|
|
})
|
|
run := store.Run{ID: runID, OrgID: "org", Workflow: "agent", Version: 1, Input: jsontext.Value(`null`)}
|
|
engine.execute(ctx, run, uuid.New())
|
|
}
|
|
|
|
var runID = uuid.MustParse("6b1f6c2e-8d5a-4c43-9f0e-1d2a3b4c5d6e")
|