Files
archie/internal/tools/budget_test.go
T

506 lines
13 KiB
Go

package tools
import (
"os"
"path/filepath"
"strings"
"testing"
"unicode/utf8"
)
func TestDefaultTurnBudgetChars(t *testing.T) {
if DefaultTurnBudgetChars != 200_000 {
t.Errorf("DefaultTurnBudgetChars = %d, want 200000", DefaultTurnBudgetChars)
}
}
func TestTurnBudgetConsume(t *testing.T) {
b := NewTurnBudget(1000, "")
// Consume within budget.
if !b.Consume(500) {
t.Error("expected consume to succeed")
}
if b.Used() != 500 {
t.Errorf("Used = %d, want 500", b.Used())
}
if b.Remaining() != 500 {
t.Errorf("Remaining = %d, want 500", b.Remaining())
}
if b.Exceeded() {
t.Error("should not be exceeded")
}
// Consume more within budget.
if !b.Consume(400) {
t.Error("expected consume to succeed")
}
// This exceeds the budget.
if b.Consume(200) {
t.Error("expected consume to fail")
}
if !b.Exceeded() {
t.Error("should be exceeded")
}
if b.Remaining() != 0 {
t.Errorf("Remaining = %d, want 0", b.Remaining())
}
// Subsequent consumes always fail.
if b.Consume(1) {
t.Error("consume should fail after exceeded")
}
}
func TestTurnBudgetExactBoundary(t *testing.T) {
// At exactly MaxChars, should succeed. Exceeded only above MaxChars.
b := NewTurnBudget(100, "")
if !b.Consume(99) {
t.Error("99/100 should succeed")
}
if !b.Consume(1) {
t.Error("100/100 should succeed (exactly at limit)")
}
if b.Exceeded() {
t.Error("should not be exceeded at exactly the limit")
}
// One more char triggers exceeded.
if b.Consume(1) {
t.Error("101/100 should be exceeded")
}
if !b.Exceeded() {
t.Error("should be exceeded above the limit")
}
}
func TestTurnBudgetSpillToDisk(t *testing.T) {
dir := t.TempDir()
b := NewTurnBudget(100, dir)
// Spill some content.
content := []byte(strings.Repeat("x", 200))
ref := b.Spill("my-tool", content)
if ref.SizeChars != 200 {
t.Errorf("SizeChars = %d, want 200", ref.SizeChars)
}
if ref.Path == "" {
t.Error("spill path should not be empty")
}
if !strings.HasPrefix(ref.Path, dir) {
t.Errorf("spill path %q should be under %q", ref.Path, dir)
}
// Verify file exists and has correct content.
got, err := os.ReadFile(ref.Path)
if err != nil {
t.Fatalf("read spill file: %v", err)
}
if string(got) != string(content) {
t.Errorf("spill content mismatch: got %q, want %q", string(got), string(content))
}
// Budget should be exceeded after spill.
if !b.Exceeded() {
t.Error("budget should be exceeded after spilling 200 chars into 100 budget")
}
}
func TestTurnBudgetSpillNoDir(t *testing.T) {
b := NewTurnBudget(100, "")
ref := b.Spill("tool", []byte("hello"))
if ref.Path != "" {
t.Error("spill path should be empty when spillDir is empty")
}
if ref.SizeChars != 5 {
t.Errorf("SizeChars = %d, want 5", ref.SizeChars)
}
}
func TestTurnBudgetSpilled(t *testing.T) {
b := NewTurnBudget(1000, "")
b.Spill("a", []byte("aa"))
b.Spill("b", []byte("bb"))
spilled := b.Spilled()
if len(spilled) != 2 {
t.Errorf("expected 2 spills, got %d", len(spilled))
}
if spilled[0].SizeChars != 2 {
t.Errorf("first spill size = %d, want 2", spilled[0].SizeChars)
}
if spilled[1].SizeChars != 2 {
t.Errorf("second spill size = %d, want 2", spilled[1].SizeChars)
}
}
func TestTurnBudgetReset(t *testing.T) {
b := NewTurnBudget(100, "")
b.Consume(99)
b.Spill("tool", []byte("overflow"))
if !b.Exceeded() {
t.Fatal("expected exceeded")
}
b.Reset()
if b.Exceeded() {
t.Error("after reset, should not be exceeded")
}
if b.Used() != 0 {
t.Errorf("Used = %d, want 0", b.Used())
}
if b.Remaining() != 100 {
t.Errorf("Remaining = %d, want 100", b.Remaining())
}
if len(b.Spilled()) != 0 {
t.Errorf("Spilled = %d, want 0", len(b.Spilled()))
}
}
func TestTurnBudgetConcurrent(t *testing.T) {
b := NewTurnBudget(100_000, "")
done := make(chan struct{})
for range 10 {
go func() {
for range 100 {
b.Consume(10)
b.Used()
b.Remaining()
b.Exceeded()
}
done <- struct{}{}
}()
}
for range 10 {
<-done
}
// No panic and budget was consumed.
if b.Used() < 1000 {
t.Errorf("Used = %d, expected at least 1000", b.Used())
}
}
func TestTurnBudgetSpillFileNaming(t *testing.T) {
dir := t.TempDir()
b := NewTurnBudget(100_000, dir)
b.Spill("my-awesome-tool", []byte("data"))
spilled := b.Spilled()
if len(spilled) != 1 {
t.Fatalf("expected 1 spill, got %d", len(spilled))
}
// File name should contain the tool name.
base := filepath.Base(spilled[0].Path)
if !strings.Contains(base, "my-awesome-tool") {
t.Errorf("spill file name %q should contain tool name", base)
}
}
func TestCapPayload(t *testing.T) {
const (
toolName = "skill_activate"
short = "abcdefghij" // 10 bytes
)
long := strings.Repeat("x", 500)
// oversize begins with short, so a truncation at len(short) is
// recognisable in the result rather than being an opaque prefix.
oversize := short + long
tests := []struct {
name string
// budget selects how the TurnBudget is built: "none" passes nil,
// "spill" gives it a usable directory, "nospill" gives it none,
// "badspill" gives it a directory that cannot be written.
budget string
payload string
max int
wantPayload string // exact result, when the payload passes through
wantContain []string // substrings the capped result must carry
}{
{
name: "no cap when max is zero",
budget: "none",
payload: long,
max: 0,
wantPayload: long,
},
{
name: "no cap when max is negative",
budget: "none",
payload: long,
max: -1,
wantPayload: long,
},
{
name: "under the cap passes through",
budget: "none",
payload: short,
max: len(short) + 1,
wantPayload: short,
},
{
name: "exactly at the cap passes through",
budget: "none",
payload: short,
max: len(short),
wantPayload: short,
},
{
name: "over the cap without a budget truncates and names the tool",
budget: "none",
payload: oversize,
max: len(short),
wantContain: []string{short, toolName, "510"},
},
{
name: "over the cap with no spill directory truncates",
budget: "nospill",
payload: oversize,
max: len(short),
wantContain: []string{short, toolName, "510"},
},
{
name: "over the cap with a spill directory reports the path",
budget: "spill",
payload: oversize,
max: len(short),
wantContain: []string{toolName, "spill-"},
},
{
name: "spill write failure falls back to truncation",
budget: "badspill",
payload: oversize,
max: len(short),
wantContain: []string{short, toolName, "510"},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
var budget *TurnBudget
switch tc.budget {
case "spill":
budget = NewTurnBudget(1_000_000, t.TempDir())
case "nospill":
budget = NewTurnBudget(1_000_000, "")
case "badspill":
f, err := os.CreateTemp(t.TempDir(), "spill-blocker")
if err != nil {
t.Fatal(err)
}
f.Close()
budget = NewTurnBudget(1_000_000, filepath.Join(f.Name(), "sub"))
}
got := CapPayload(toolName, tc.payload, tc.max, budget)
if tc.wantPayload != "" {
if got != tc.wantPayload {
t.Errorf("CapPayload() = %q, want it returned unchanged", got)
}
return
}
for _, want := range tc.wantContain {
if !strings.Contains(got, want) {
t.Errorf("CapPayload() = %q, want it to contain %q", got, want)
}
}
if len(got) >= len(tc.payload) {
t.Errorf("CapPayload() returned %d bytes, want fewer than the %d-byte payload", len(got), len(tc.payload))
}
})
}
}
// TestCapPayloadDoesNotChargeBudget pins the accounting boundary.
//
// CapPayload displaces an oversized result to disk; it must not also bill for
// it. Spill used to charge the FULL uncapped length while the caller charged
// the capped length on top, so a turn was billed for far more than it was ever
// shown and ran out of budget early.
func TestCapPayloadDoesNotChargeBudget(t *testing.T) {
tests := []struct {
name string
spillDir bool
}{
{name: "spilled to disk", spillDir: true},
{name: "truncated inline", spillDir: false},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
dir := ""
if tc.spillDir {
dir = t.TempDir()
}
budget := NewTurnBudget(1_000_000, dir)
CapPayload("big", strings.Repeat("x", 5000), 100, budget)
if used := budget.Used(); used != 0 {
t.Errorf("Used() = %d after CapPayload, want 0: accounting belongs to the caller", used)
}
if budget.Exceeded() {
t.Error("budget reports exceeded after a single capped result")
}
})
}
}
// TestSpillChargesBudget is the other half: the batch-dispatch path calls Spill
// directly and relies on it charging, so that behaviour must not drift.
func TestSpillChargesBudget(t *testing.T) {
budget := NewTurnBudget(1_000_000, t.TempDir())
content := []byte(strings.Repeat("y", 300))
budget.Spill("tool", content)
if used := budget.Used(); used != len(content) {
t.Errorf("Used() = %d, want %d", used, len(content))
}
}
// TestWriteSpillUniquePaths guards a collision the previous naming scheme hid:
// the file name was derived from the budget's consumed count, so two spills
// that did not change it would overwrite each other.
func TestWriteSpillUniquePaths(t *testing.T) {
budget := NewTurnBudget(1_000_000, t.TempDir())
first := budget.WriteSpill("tool", []byte("first"))
second := budget.WriteSpill("tool", []byte("second"))
if first.Path == "" || second.Path == "" {
t.Fatalf("spill paths empty: %q %q", first.Path, second.Path)
}
if first.Path == second.Path {
t.Fatalf("both spills wrote to %q; the second overwrote the first", first.Path)
}
for _, want := range []struct {
ref SpillRef
body string
}{{first, "first"}, {second, "second"}} {
got, err := os.ReadFile(want.ref.Path)
if err != nil {
t.Fatal(err)
}
if string(got) != want.body {
t.Errorf("%s holds %q, want %q", want.ref.Path, got, want.body)
}
}
}
// TestWriteSpillUniqueAcrossBudgets is the collision that matters in practice.
// A TurnBudget is minted per chat turn and per agent stage while every one of
// them shares a single spill directory, so a name derived from within-budget
// state restarts each turn and the second turn silently overwrites the first --
// leaving one session holding a reference to another session's output.
func TestWriteSpillUniqueAcrossBudgets(t *testing.T) {
dir := t.TempDir()
firstTurn := NewTurnBudget(1_000_000, dir)
secondTurn := NewTurnBudget(1_000_000, dir)
a := firstTurn.WriteSpill("read", []byte("first turn output"))
b := secondTurn.WriteSpill("read", []byte("second turn output"))
if a.Path == "" || b.Path == "" {
t.Fatalf("spill paths empty: %q %q", a.Path, b.Path)
}
if a.Path == b.Path {
t.Fatalf("both turns wrote to %q", a.Path)
}
got, err := os.ReadFile(a.Path)
if err != nil {
t.Fatal(err)
}
if string(got) != "first turn output" {
t.Errorf("first turn's spill holds %q; a later turn overwrote it", got)
}
}
// TestWriteSpillFilePermissions checks the mode on the first place archie
// persists verbatim tool output. It can hold anything the tools could read --
// file contents, shell stdout, MCP responses -- so it must not be world- or
// group-readable.
func TestWriteSpillFilePermissions(t *testing.T) {
budget := NewTurnBudget(1_000_000, t.TempDir())
ref := budget.WriteSpill("shell", []byte("secret output"))
if ref.Path == "" {
t.Fatal("spill was not written")
}
info, err := os.Stat(ref.Path)
if err != nil {
t.Fatal(err)
}
if perm := info.Mode().Perm(); perm != 0o600 {
t.Errorf("spill file mode = %04o, want 0600", perm)
}
}
// TestWriteSpillSanitisesToolName covers a name reaching the filesystem. MCP
// tool names come from a third-party server, so a name carrying path
// separators is reachable input rather than a hypothetical.
func TestWriteSpillSanitisesToolName(t *testing.T) {
dir := t.TempDir()
budget := NewTurnBudget(1_000_000, dir)
ref := budget.WriteSpill("../../etc/passwd", []byte("payload"))
if ref.Path == "" {
t.Fatal("spill was not written")
}
if filepath.Dir(ref.Path) != dir {
t.Errorf("spill escaped to %q, want it inside %q", ref.Path, dir)
}
}
// TestCapPayloadKeepsRunesIntact guards the truncation boundary: cutting a
// multi-byte rune in half hands the model invalid UTF-8, which some providers
// reject outright and others render as a replacement character mid-word.
func TestCapPayloadKeepsRunesIntact(t *testing.T) {
payload := strings.Repeat("é", 100) // 2 bytes per rune
for limit := 1; limit <= 20; limit++ {
got := CapPayload("read", payload, limit, nil)
if !utf8.ValidString(got) {
t.Errorf("limit=%d: CapPayload() returned invalid UTF-8: %q", limit, got)
}
}
}
func TestTurnBudgetSpillDirPermissions(t *testing.T) {
// Create a regular file and use a sub-path as the spill directory.
// Even root cannot write a file when a path component is a regular
// file, making this test reliable regardless of privileges.
f, err := os.CreateTemp(t.TempDir(), "archie-test-spill-blocker")
if err != nil {
t.Fatal(err)
}
f.Close()
defer os.Remove(f.Name())
// spillDir is a sub-path of the regular file — WriteFile must fail.
spillDir := filepath.Join(f.Name(), "sub")
b := NewTurnBudget(100_000, spillDir)
ref := b.Spill("tool", []byte("data"))
// Spill should handle write failure gracefully.
if ref.Path != "" {
t.Errorf("spill path should be empty on write failure, got %q", ref.Path)
}
if ref.SizeChars != 4 {
t.Errorf("SizeChars should still be recorded: got %d", ref.SizeChars)
}
}