Files
archie/internal/agentexec/toolset_test.go
T
2026-08-06 01:40:15 +10:00

474 lines
14 KiB
Go

package agentexec
import (
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"strings"
"testing"
aicore "github.com/samcharles93/ai-sdk/core"
"github.com/samcharles93/archie-core/internal/tools"
)
func TestToolSetConvertsRegisteredEntries(t *testing.T) {
reg := tools.NewRegistry()
if err := reg.Register(tools.ToolEntry{
Name: "echo",
Description: "echoes its input",
Schema: tools.JSONSchema{"type": "object", "properties": map[string]any{"msg": map[string]any{"type": "string"}}},
Handler: func(_ context.Context, input map[string]any) (any, error) {
return input["msg"], nil
},
}); err != nil {
t.Fatal(err)
}
set := mustBuildToolSet(t, reg)
tool, ok := set["echo"]
if !ok {
t.Fatal("ToolSet did not include the \"echo\" entry")
}
if tool.Name != "echo" || tool.Description != "echoes its input" {
t.Errorf("tool = %+v", tool)
}
if len(tool.Parameters) == 0 {
t.Error("Parameters is empty, want the marshaled schema")
}
}
func TestToolSetHandlerRoundTripsJSON(t *testing.T) {
reg := tools.NewRegistry()
if err := reg.Register(tools.ToolEntry{
Name: "add",
Handler: func(_ context.Context, input map[string]any) (any, error) {
a, _ := input["a"].(float64)
b, _ := input["b"].(float64)
return a + b, nil
},
}); err != nil {
t.Fatal(err)
}
set := mustBuildToolSet(t, reg)
out, err := set["add"].Execute(context.Background(), `{"a": 2, "b": 3}`)
if err != nil {
t.Fatal(err)
}
var got float64
if err := json.Unmarshal([]byte(out), &got); err != nil {
t.Fatalf("output %q did not decode as JSON: %v", out, err)
}
if got != 5 {
t.Errorf("result = %v, want 5", got)
}
}
func TestToolSetHandlerPropagatesHandlerError(t *testing.T) {
reg := tools.NewRegistry()
wantErr := errors.New("boom")
if err := reg.Register(tools.ToolEntry{
Name: "fails",
Handler: func(context.Context, map[string]any) (any, error) {
return nil, wantErr
},
}); err != nil {
t.Fatal(err)
}
set := mustBuildToolSet(t, reg)
_, err := set["fails"].Execute(context.Background(), `{}`)
if err == nil {
t.Fatal("expected error from failing handler")
}
}
func TestToolSetHandlerRejectsMalformedInput(t *testing.T) {
reg := tools.NewRegistry()
if err := reg.Register(tools.ToolEntry{
Name: "noop",
Handler: func(context.Context, map[string]any) (any, error) { return "ok", nil },
}); err != nil {
t.Fatal(err)
}
set := mustBuildToolSet(t, reg)
_, err := set["noop"].Execute(context.Background(), `not json`)
if err == nil {
t.Fatal("expected error decoding malformed input JSON")
}
}
func TestToolSetOmitsUnavailableTools(t *testing.T) {
reg := tools.NewRegistry()
if err := reg.Register(tools.ToolEntry{
Name: "offline",
Handler: func(context.Context, map[string]any) (any, error) { return nil, nil },
CheckFn: func() bool { return false },
}); err != nil {
t.Fatal(err)
}
set := mustBuildToolSet(t, reg)
if _, ok := set["offline"]; ok {
t.Error("ToolSet included a tool whose CheckFn reports unavailable")
}
}
func TestToolSetEmptyRegistryReturnsEmptySet(t *testing.T) {
set := mustBuildToolSet(t, tools.NewRegistry())
if len(set) != 0 {
t.Errorf("len(set) = %d, want 0", len(set))
}
}
func TestToolSetNilRegistryReturnsEmptySet(t *testing.T) {
set := mustBuildToolSet(t, nil)
if len(set) != 0 {
t.Errorf("len(set) = %d, want 0 for nil registry", len(set))
}
}
func TestToolSetUsesObjectSchemaForParameterlessTools(t *testing.T) {
reg := tools.NewRegistry()
if err := reg.Register(tools.ToolEntry{
Name: "no_args",
Handler: func(context.Context, map[string]any) (any, error) { return nil, nil },
}); err != nil {
t.Fatal(err)
}
parameters := mustBuildToolSet(t, reg)["no_args"].Parameters
var schema map[string]any
if err := json.Unmarshal(parameters, &schema); err != nil {
t.Fatalf("Parameters = %s, want JSON object: %v", parameters, err)
}
if schema["type"] != "object" {
t.Fatalf("Parameters = %s, want object schema", parameters)
}
}
func TestToolSetHandlerReturningNilOutputProducesNullJSON(t *testing.T) {
reg := tools.NewRegistry()
if err := reg.Register(tools.ToolEntry{
Name: "voidy",
Handler: func(context.Context, map[string]any) (any, error) { return nil, nil },
}); err != nil {
t.Fatal(err)
}
set := mustBuildToolSet(t, reg)
out, err := set["voidy"].Execute(context.Background(), `{}`)
if err != nil {
t.Fatal(err)
}
if out != "null" {
t.Errorf("output = %q, want \"null\"", out)
}
}
func TestBuildToolSetReportsInvalidDynamicSchemas(t *testing.T) {
tests := []struct {
name string
entry tools.ToolEntry
}{
{
name: "schema override panic",
entry: tools.ToolEntry{
Name: "panics",
Handler: func(context.Context, map[string]any) (any, error) { return nil, nil },
DynamicSchemaOverrides: func(tools.JSONSchema) tools.JSONSchema {
panic("schema panic")
},
},
},
{
name: "unmarshalable schema",
entry: tools.ToolEntry{
Name: "invalid",
Schema: tools.JSONSchema{"bad": func() {}},
Handler: func(context.Context, map[string]any) (any, error) { return nil, nil },
},
},
{
name: "availability panic",
entry: tools.ToolEntry{
Name: "unavailable",
Handler: func(context.Context, map[string]any) (any, error) { return nil, nil },
CheckFn: func() bool {
panic("availability panic")
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
registry := tools.NewRegistry()
if err := registry.Register(tt.entry); err != nil {
t.Fatal(err)
}
var (
err error
panicked any
)
func() {
defer func() { panicked = recover() }()
_, err = BuildToolSet(registry, ToolSetOptions{})
}()
if panicked != nil {
t.Fatalf("BuildToolSet() panicked: %v", panicked)
}
if err == nil {
t.Fatal("BuildToolSet() error = nil, want invalid schema error")
}
})
}
}
func mustBuildToolSet(t *testing.T, registry *tools.Registry) map[string]*aicore.Tool {
t.Helper()
return mustBuildToolSetWith(t, registry, ToolSetOptions{})
}
func mustBuildToolSetWith(t *testing.T, registry *tools.Registry, opts ToolSetOptions) map[string]*aicore.Tool {
t.Helper()
set, err := BuildToolSet(registry, opts)
if err != nil {
t.Fatal(err)
}
return set
}
// registryReturning builds a single-entry registry whose tool returns payload.
func registryReturning(t *testing.T, entry tools.ToolEntry, payload string) *tools.Registry {
t.Helper()
reg := tools.NewRegistry()
entry.Handler = func(context.Context, map[string]any) (any, error) { return payload, nil }
if err := reg.Register(entry); err != nil {
t.Fatal(err)
}
return reg
}
// TestToolSetCapsResults covers the limit that decides how much of a tool
// result reaches the model. The workspace tools truncate themselves, so the
// entries that matter here are the ones that do not: skill bodies and MCP
// results were previously handed over whole.
func TestToolSetCapsResults(t *testing.T) {
const payloadSize = 4000
payload := strings.Repeat("z", payloadSize)
tests := []struct {
name string
entryLimit int // ToolEntry.MaxResultSizeChars
defaultLimit int // ToolSetOptions.MaxResultChars
wantTruncated bool
}{
{name: "under the default passes through", defaultLimit: payloadSize * 2},
{name: "over the default is capped", defaultLimit: 100, wantTruncated: true},
{name: "per-entry limit overrides a larger default", entryLimit: 100, defaultLimit: payloadSize * 2, wantTruncated: true},
{name: "per-entry limit overrides a smaller default", entryLimit: payloadSize * 2, defaultLimit: 100},
{name: "negative default disables capping", defaultLimit: -1},
{name: "zero default disables capping", defaultLimit: 0},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
reg := registryReturning(t, tools.ToolEntry{
Name: "big",
MaxResultSizeChars: tc.entryLimit,
}, payload)
set := mustBuildToolSetWith(t, reg, ToolSetOptions{MaxResultChars: tc.defaultLimit})
out, err := set["big"].Execute(context.Background(), "{}")
if err != nil {
t.Fatal(err)
}
truncated := strings.Contains(out, "truncated")
if truncated != tc.wantTruncated {
t.Fatalf("truncated = %v, want %v (result %d bytes)", truncated, tc.wantTruncated, len(out))
}
if !tc.wantTruncated && len(out) < payloadSize {
t.Errorf("result is %d bytes, want the whole %d-byte payload", len(out), payloadSize)
}
})
}
}
// TestToolSetSpillsOversizeResults proves an oversized result is displaced to
// disk rather than lost: the model gets a path it can read back with the read
// tool, which is the whole point of spilling instead of truncating.
func TestToolSetSpillsOversizeResults(t *testing.T) {
dir := t.TempDir()
budget := tools.NewTurnBudget(1_000_000, dir)
payload := strings.Repeat("q", 4000)
reg := registryReturning(t, tools.ToolEntry{Name: "big"}, payload)
set := mustBuildToolSetWith(t, reg, ToolSetOptions{MaxResultChars: 100, Budget: budget})
out, err := set["big"].Execute(context.Background(), "{}")
if err != nil {
t.Fatal(err)
}
if !strings.Contains(out, "spilled") {
t.Fatalf("result = %q, want a spill reference", out)
}
spilled := budget.Spilled()
if len(spilled) != 1 || spilled[0].Path == "" {
t.Fatalf("Spilled() = %+v, want one spill with a path", spilled)
}
if !strings.Contains(out, spilled[0].Path) {
t.Errorf("result %q does not name the spill path %q", out, spilled[0].Path)
}
body, err := os.ReadFile(spilled[0].Path)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(body), payload) {
t.Error("spill file does not hold the full payload")
}
}
// TestToolLimitsCreatesSpillDir covers the gap that made spilling dead code in
// production: the configured directory defaults to <work_dir>/tool-spill and
// nothing ever created it, while TurnBudget.Spill swallows the write error. So
// every oversized result silently fell back to inline truncation and the spill
// path was never exercised outside tests that made their own directory.
func TestToolLimitsCreatesSpillDir(t *testing.T) {
// A nested path proves the whole tree is created, not just a leaf.
dir := filepath.Join(t.TempDir(), "work", "tool-spill")
limits := ToolLimits{MaxResultChars: 100, TurnBudgetChars: 1_000_000, SpillDir: dir}
if err := limits.EnsureSpillDir(); err != nil {
t.Fatal(err)
}
info, err := os.Stat(dir)
if err != nil {
t.Fatalf("spill directory was not created: %v", err)
}
if !info.IsDir() {
t.Fatalf("%s is not a directory", dir)
}
// And a spill actually lands there once it exists.
reg := registryReturning(t, tools.ToolEntry{Name: "big"}, strings.Repeat("z", 5000))
set := mustBuildToolSetWith(t, reg, limits.Options())
if _, err := set["big"].Execute(context.Background(), "{}"); err != nil {
t.Fatal(err)
}
entries, err := os.ReadDir(dir)
if err != nil {
t.Fatal(err)
}
if len(entries) == 0 {
t.Error("no spill file was written into the created directory")
}
}
// TestToolLimitsEnsureSpillDirNoop covers the disabled case: no directory
// configured is not an error, it selects inline truncation.
func TestToolLimitsEnsureSpillDirNoop(t *testing.T) {
if err := (ToolLimits{}).EnsureSpillDir(); err != nil {
t.Errorf("EnsureSpillDir() with no directory = %v, want nil", err)
}
}
// TestToolSetChargesBudgetOnce pins the invariant that makes the turn budget
// mean anything: a turn is charged exactly the bytes it was shown.
//
// Displacing an oversized result to disk used to charge the full displaced
// size AND the short reference handed back, so two 60 KB results burned more
// than the 200 K default budget while the model saw about 200 characters.
func TestToolSetChargesBudgetOnce(t *testing.T) {
tests := []struct {
name string
spillDir bool
}{
{name: "results spilled to disk", spillDir: true},
{name: "results truncated inline", spillDir: false},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
dir := ""
if tc.spillDir {
dir = t.TempDir()
}
budget := tools.NewTurnBudget(1_000_000, dir)
reg := registryReturning(t, tools.ToolEntry{Name: "big"}, strings.Repeat("q", 60_000))
set := mustBuildToolSetWith(t, reg, ToolSetOptions{MaxResultChars: 100, Budget: budget})
delivered := 0
for range 2 {
out, err := set["big"].Execute(context.Background(), "{}")
if err != nil {
t.Fatal(err)
}
delivered += len(out)
}
if used := budget.Used(); used != delivered {
t.Errorf("budget charged %d for %d delivered characters", used, delivered)
}
})
}
}
// TestToolSetBudgetStopsFurtherCalls checks the aggregate cap: once a turn has
// spent its budget the next tool call must fail loudly, so the model is told it
// has run out rather than silently receiving nothing.
func TestToolSetBudgetStopsFurtherCalls(t *testing.T) {
budget := tools.NewTurnBudget(50, "")
reg := registryReturning(t, tools.ToolEntry{Name: "chatty"}, strings.Repeat("w", 200))
set := mustBuildToolSetWith(t, reg, ToolSetOptions{Budget: budget})
if _, err := set["chatty"].Execute(context.Background(), "{}"); err != nil {
t.Fatalf("first call: %v", err)
}
if !budget.Exceeded() {
t.Fatal("budget should be exceeded after a 200-byte result against a 50-byte cap")
}
_, err := set["chatty"].Execute(context.Background(), "{}")
if err == nil {
t.Fatal("second call succeeded, want a budget-exceeded error")
}
if !strings.Contains(err.Error(), "budget") {
t.Errorf("error = %v, want it to name the budget", err)
}
}
// TestToolSetExcludesToolsets covers the task-agent case: a task runs in its
// own worktree, so the chat workspace's file and shell tools must not follow it
// in and bypass the agent loop's read-only and protected-path enforcement.
func TestToolSetExcludesToolsets(t *testing.T) {
reg := tools.NewRegistry()
for _, e := range []tools.ToolEntry{
{Name: "shell", Toolset: "workspace"},
{Name: "memory_edit", Toolset: "memory"},
} {
e.Handler = func(context.Context, map[string]any) (any, error) { return "ok", nil }
if err := reg.Register(e); err != nil {
t.Fatal(err)
}
}
full := mustBuildToolSetWith(t, reg, ToolSetOptions{})
if len(full) != 2 {
t.Fatalf("with no exclusions the set has %d tools, want 2", len(full))
}
limited := mustBuildToolSetWith(t, reg, ToolSetOptions{ExcludeToolsets: []string{"workspace"}})
if _, ok := limited["shell"]; ok {
t.Error("workspace tool survived the exclusion")
}
if _, ok := limited["memory_edit"]; !ok {
t.Error("memory tool was dropped, want only the workspace toolset excluded")
}
}