Also fix repository-free message.send validation (#2046) and add a single-binary development smoke path (#1977). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
240 lines
8.8 KiB
Go
240 lines
8.8 KiB
Go
package runs
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json/jsontext"
|
|
json "encoding/json/v2"
|
|
"errors"
|
|
"log/slog"
|
|
"os"
|
|
"slices"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/samcharles93/ai-sdk/chat"
|
|
"github.com/samcharles93/ai-sdk/core"
|
|
|
|
"github.com/samcharles93/archie-core/modules/governance"
|
|
"github.com/samcharles93/archie-core/modules/runs/internal/store"
|
|
)
|
|
|
|
type triageModel struct {
|
|
scriptedModel
|
|
models []string
|
|
}
|
|
|
|
func (m *triageModel) Chat(_ context.Context, req chat.Request) (chat.Response, error) {
|
|
m.models = append(m.models, req.Model)
|
|
if req.Messages[len(req.Messages)-1].Role == chat.RoleTool {
|
|
return chat.Response{Role: chat.RoleAssistant, Content: "No change needed.", FinishReason: "stop"}, nil
|
|
}
|
|
return chat.Response{
|
|
Role: chat.RoleAssistant, FinishReason: "tool_calls",
|
|
ToolCalls: []chat.ToolCall{{ID: "classification", Name: "return_result", Arguments: `{"needs_code_change":false,"reasons":"Already resolved"}`}},
|
|
}, nil
|
|
}
|
|
|
|
type modelSettings struct {
|
|
onePersonPath
|
|
model *triageModel
|
|
name string
|
|
}
|
|
|
|
func (s modelSettings) AgentModelForAlias(_ context.Context, alias string) (governance.Model, error) {
|
|
if alias != "fast" {
|
|
return governance.Model{}, governance.ErrNoAgentModel
|
|
}
|
|
return governance.Model{Model: s.name}, nil
|
|
}
|
|
|
|
func (s modelSettings) AgentProvider(context.Context, governance.Model) (chat.Provider, chat.ModelInfo, error) {
|
|
return s.model, chat.ModelInfo{}, nil
|
|
}
|
|
|
|
func (modelSettings) AttemptsPerRun(context.Context) (int, error) { return 6, nil }
|
|
|
|
// TestTriageClassificationUsesPinnedModel checks that a resumed step returns
|
|
// the same typed classification even if its model alias now resolves differently.
|
|
func TestTriageClassificationUsesPinnedModel(t *testing.T) {
|
|
raw, err := os.ReadFile("testdata/corpus/triage.yaml")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
body, _, ok := strings.Cut(string(raw), " - type: forge.comment")
|
|
if !ok {
|
|
t.Fatal("triage fixture has no forge comment step")
|
|
}
|
|
body = strings.Replace(body, " read_only: true", " read_only: true\n model: fast", 1)
|
|
doc, err := ParseDocument(body)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
engine := newEngine(nil, &fakeWork{}, slog.New(slog.DiscardHandler), Options{})
|
|
engine.Use(Deps{Settings: modelSettings{}})
|
|
_, err = engine.Install(t.Context(), "org", "triage", strings.Replace(body, "model: fast", "model: missing", 1), "operator")
|
|
var invalid *InvalidError
|
|
if !errors.As(err, &invalid) || len(invalid.Problems) != 1 || invalid.Problems[0].Path != "steps[0].settings.model" {
|
|
t.Fatalf("saving an unknown alias returned %v, want a model setting error", err)
|
|
}
|
|
run := store.Run{ID: uuid.New()}
|
|
work := &fakeWork{}
|
|
aliases, withMemory := agentUse(doc)
|
|
for _, tt := range []struct {
|
|
name string
|
|
want []string
|
|
}{
|
|
{name: "fast-v1", want: []string{"fast-v1", "fast-v1"}},
|
|
{name: "fast-v2"},
|
|
} {
|
|
model := &triageModel{}
|
|
deps := Deps{Settings: modelSettings{model: model, name: tt.name}, Toolbox: anyBox{}, Judge: &actionJudge{}}
|
|
rc := newContext(t.Context(), work, run, uuid.New(), work.steps)
|
|
p := &progress{results: make(map[string]StepResult)}
|
|
if err := deps.pin(rc, p, aliases, withMemory); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := deps.runStep(rc, p, doc.Steps[0], 0); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got := p.results["classify"].Result
|
|
if got["needs_code_change"] != false || got["reasons"] != "Already resolved" {
|
|
t.Fatalf("classification = %#v", got)
|
|
}
|
|
if !slices.Equal(model.models, tt.want) {
|
|
t.Fatalf("model calls = %v, want %v", model.models, tt.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// scriptedModel answers by turn: a tool call first, then findings once the
|
|
// tool's result is in the history.
|
|
type scriptedModel struct {
|
|
mu sync.Mutex
|
|
calls int
|
|
}
|
|
|
|
func (m *scriptedModel) Name() string { return "scripted" }
|
|
|
|
func (m *scriptedModel) Chat(_ context.Context, req chat.Request) (chat.Response, error) {
|
|
m.mu.Lock()
|
|
m.calls++
|
|
m.mu.Unlock()
|
|
last := req.Messages[len(req.Messages)-1]
|
|
if last.Role == chat.RoleTool {
|
|
return chat.Response{
|
|
Role: chat.RoleAssistant, Content: "disk is full: " + last.Content, FinishReason: "stop",
|
|
Parts: chat.Parts{chat.ReasoningPart{Text: "the tool said why"}, chat.TextPart{Text: "disk is full: " + last.Content}},
|
|
Usage: chat.Usage{PromptTokens: 20, CompletionTokens: 5, TotalTokens: 25},
|
|
}, nil
|
|
}
|
|
return chat.Response{
|
|
Role: chat.RoleAssistant, FinishReason: "tool_calls",
|
|
Parts: chat.Parts{chat.ReasoningPart{Text: "look at the disk"}},
|
|
ToolCalls: []chat.ToolCall{{ID: "call-1", Name: "disk_usage", Arguments: `{"path":"/tmp"}`}},
|
|
Usage: chat.Usage{PromptTokens: 10, CompletionTokens: 3, TotalTokens: 13},
|
|
}, nil
|
|
}
|
|
|
|
func (m *scriptedModel) ChatStream(context.Context, chat.Request) (chat.Stream, error) {
|
|
return nil, errors.New("not streamed")
|
|
}
|
|
|
|
// TestAgentResumesFromItsRecordedCalls guards the agent loop's checkpoint
|
|
// contract (2.2, 2.3): a worker that stops after any recorded call, killed or
|
|
// shut down mid-tool, resumes the loop from the run's steps, replays what was
|
|
// recorded without calling the model or running the tool again, and ends with
|
|
// the same findings. A step encoding that changes a recorded call shows up as
|
|
// a replay that diverges from its journal.
|
|
func TestAgentResumesFromItsRecordedCalls(t *testing.T) {
|
|
// The uninterrupted run records a step before the loop (seq 0), the
|
|
// loop's model, tool and model calls (1 to 3), and a step after it (4).
|
|
type stop struct {
|
|
name string
|
|
// killAt makes writes fail as a lost lease from that many steps on.
|
|
killAt *int
|
|
// shutdownInTool ends the run's context while the tool runs.
|
|
shutdownInTool bool
|
|
// replayedModel and replayedTool are the calls the resume replays.
|
|
replayedModel, replayedTool int
|
|
}
|
|
at := func(n int) *int { return &n }
|
|
tests := []stop{
|
|
{name: "killed before the loop", killAt: at(0)},
|
|
{name: "killed before the first model call is recorded", killAt: at(1)},
|
|
{name: "killed before the tool is recorded", killAt: at(2), replayedModel: 1},
|
|
{name: "killed before the second model call is recorded", killAt: at(3), replayedModel: 1, replayedTool: 1},
|
|
{name: "killed before the last step", killAt: at(4), replayedModel: 2, replayedTool: 1},
|
|
{name: "killed before finishing", killAt: at(5), replayedModel: 2, replayedTool: 1},
|
|
{name: "shut down while the tool runs", shutdownInTool: true, replayedModel: 1, replayedTool: 1},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
ctx, shutdown := context.WithCancel(t.Context())
|
|
defer shutdown()
|
|
duringTool := func() {}
|
|
if tt.shutdownInTool {
|
|
duringTool = shutdown
|
|
}
|
|
first := &fakeWork{rejectFrom: tt.killAt}
|
|
execute(ctx, t, first, &scriptedModel{}, new(int), duringTool)
|
|
|
|
resumed := &fakeWork{steps: first.steps}
|
|
model, toolRuns := &scriptedModel{}, new(int)
|
|
execute(t.Context(), t, resumed, model, toolRuns, func() {})
|
|
|
|
if len(resumed.finished) != 1 || resumed.finished[0] != store.StateSucceeded {
|
|
t.Fatalf("resumed run finished %v, want succeeded", resumed.finished)
|
|
}
|
|
var res AgentResult
|
|
if err := json.Unmarshal(resumed.results[0], &res); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if res.Answer != "disk is full: 93%" || res.Usage.TotalTokens != 38 {
|
|
t.Fatalf("resumed findings = %+v", res)
|
|
}
|
|
if want := 1 - tt.replayedTool; *toolRuns != want {
|
|
t.Fatalf("tool ran %d times on resume, want %d", *toolRuns, want)
|
|
}
|
|
if want := 2 - tt.replayedModel; model.calls != want {
|
|
t.Fatalf("model called %d times on resume, want %d", model.calls, want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// execute runs the probe workflow once on work. duringTool runs inside the
|
|
// tool call.
|
|
func execute(ctx context.Context, t *testing.T, work *fakeWork, model chat.Provider, toolRuns *int, duringTool func()) {
|
|
t.Helper()
|
|
engine := newEngine(nil, work, slog.New(slog.DiscardHandler), Options{})
|
|
useWorkflow(engine, func(rc *Context, _ jsontext.Value) (jsontext.Value, error) {
|
|
if _, err := rc.Step("before", func(context.Context, string) (bool, error) { return true, nil }); err != nil {
|
|
return nil, err
|
|
}
|
|
res, err := rc.Agent(AgentRequest{
|
|
Name: "probe", Provider: model, Model: "m", Prompt: "why is the disk full?",
|
|
Tools: core.ToolSet{"disk_usage": {
|
|
Name: "disk_usage", Parameters: []byte(`{"type":"object"}`),
|
|
Execute: func(ctx context.Context, input string) (string, error) {
|
|
if !strings.Contains(core.IdempotencyKey(ctx), ":probe:") {
|
|
return "", errors.New("no idempotency key")
|
|
}
|
|
*toolRuns++
|
|
duringTool()
|
|
return "93%", nil
|
|
},
|
|
}},
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return rc.Step("after", func(context.Context, string) (jsontext.Value, error) { return json.Marshal(res) })
|
|
})
|
|
run := store.Run{ID: runID, OrgID: "org", Workflow: "agent", Version: 1, Input: jsontext.Value(`null`)}
|
|
engine.execute(ctx, run, uuid.New())
|
|
}
|
|
|
|
var runID = uuid.MustParse("6b1f6c2e-8d5a-4c43-9f0e-1d2a3b4c5d6e")
|