Files
archie/internal/agentexec/harness_output_claude_test.go
T
sam 7a54db0cc6 fix(agentexec): count Claude cache tokens as prompt tokens, as the built-in loop does
Claude's input_tokens excludes cache reads and writes; the built-in
loop's Anthropic provider folds them into PromptTokens so budgets count
total prompt size. The harness adapter now does the same, so one budget
means the same on either runner.
2026-09-26 09:11:57 +10:00

149 lines
5.2 KiB
Go

package agentexec
import (
"bufio"
"encoding/json"
"os"
"reflect"
"strings"
"testing"
)
func TestClaudeCodeHarnessOutputFixtures(t *testing.T) {
errorDetail := "Exit code 1\ncat: /nonexistent-file: No such file or directory"
cases := []struct {
name string
wantCalls []ToolCallReport
wantUsage Usage
wantSession string
}{
{
name: "run.jsonl",
wantCalls: []ToolCallReport{
{Tool: "Bash", Detail: "hello"},
{Tool: "Bash", Detail: errorDetail, Failed: true},
},
wantUsage: Usage{PromptTokens: 102420, CompletionTokens: 248, TotalTokens: 102668, CachedTokens: 78105, CacheCreationTokens: 24309},
wantSession: "9c22c15c-80ef-42c6-a503-8d087ec47d92",
},
{
name: "killed-before-result.jsonl",
wantCalls: []ToolCallReport{
{Tool: "Bash", Detail: "hello"},
{Tool: "Bash", Detail: errorDetail, Failed: true},
},
wantUsage: Usage{PromptTokens: 102420, CompletionTokens: 121, TotalTokens: 102541, CachedTokens: 78105, CacheCreationTokens: 24309},
wantSession: "9c22c15c-80ef-42c6-a503-8d087ec47d92",
},
{
name: "malformed-line.jsonl",
wantCalls: []ToolCallReport{
{Tool: "Bash", Detail: "hello"},
{Tool: "Bash", Detail: errorDetail, Failed: true},
},
wantUsage: Usage{PromptTokens: 102420, CompletionTokens: 248, TotalTokens: 102668, CachedTokens: 78105, CacheCreationTokens: 24309},
wantSession: "9c22c15c-80ef-42c6-a503-8d087ec47d92",
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
stream, err := os.Open("testdata/claude/" + tc.name)
if err != nil {
t.Fatal(err)
}
defer stream.Close()
output := newClaudeCodeHarnessOutput()
var calls []ToolCallReport
scanner := bufio.NewScanner(stream)
for scanner.Scan() {
output.Line(scanner.Bytes(), func(call ToolCallReport) { calls = append(calls, call) })
}
if err := scanner.Err(); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(calls, tc.wantCalls) {
t.Errorf("tool calls = %#v, want %#v", calls, tc.wantCalls)
}
if got := output.SessionID(); got != tc.wantSession {
t.Errorf("session ID = %q, want %q", got, tc.wantSession)
}
if got := output.Usage(); got != tc.wantUsage {
t.Errorf("usage = %#v, want %#v", got, tc.wantUsage)
}
})
}
}
func TestClaudeCodeHarnessOutputToolResultContent(t *testing.T) {
long := strings.Repeat("x", toolCallDetailBytes+25)
cases := []struct {
name string
content any
wantDetail string
}{
{name: "string", content: "plain result", wantDetail: "plain result"},
{name: "text blocks", content: []map[string]string{{"type": "text", "text": "first"}, {"type": "text", "text": "second"}}, wantDetail: "first\nsecond"},
{name: "clipped", content: long, wantDetail: strings.Repeat("x", toolCallDetailBytes) + "…"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
output := newClaudeCodeHarnessOutput()
var calls []ToolCallReport
report := func(call ToolCallReport) { calls = append(calls, call) }
sendClaudeEvent(t, output, report, map[string]any{
"type": "assistant", "session_id": "old-session",
"message": map[string]any{
"id": "same-message", "usage": claudeUsageFixture(2, 3, 5, 7),
"content": []any{map[string]any{"type": "tool_use", "id": "call-1", "name": "Read", "input": map[string]any{}}},
},
})
sendClaudeEvent(t, output, report, map[string]any{
"type": "assistant", "session_id": "newer-session",
"message": map[string]any{
"id": "same-message", "usage": claudeUsageFixture(2, 3, 5, 7),
"content": []any{map[string]any{"type": "tool_use", "id": "call-2", "name": "Write", "input": map[string]any{}}},
},
})
sendClaudeEvent(t, output, report, map[string]any{
"type": "user", "session_id": "latest-session",
"message": map[string]any{"content": []any{map[string]any{"type": "tool_result", "tool_use_id": "call-1", "content": tc.content, "is_error": false}}},
})
sendClaudeEvent(t, output, report, map[string]any{
"type": "user", "session_id": "",
"message": map[string]any{"content": []any{map[string]any{"type": "tool_result", "tool_use_id": "call-2", "content": "done", "is_error": false}}},
})
wantCalls := []ToolCallReport{{Tool: "Read", Detail: tc.wantDetail}, {Tool: "Write", Detail: "done"}}
if !reflect.DeepEqual(calls, wantCalls) {
t.Errorf("tool calls = %#v, want %#v", calls, wantCalls)
}
wantUsage := Usage{PromptTokens: 14, CompletionTokens: 3, TotalTokens: 17, CachedTokens: 5, CacheCreationTokens: 7}
if got := output.Usage(); got != wantUsage {
t.Errorf("usage = %#v, want %#v", got, wantUsage)
}
if got := output.SessionID(); got != "latest-session" {
t.Errorf("session ID = %q, want latest non-empty event value", got)
}
})
}
}
func sendClaudeEvent(t *testing.T, output HarnessOutput, report ToolCallReporter, event map[string]any) {
t.Helper()
line, err := json.Marshal(event)
if err != nil {
t.Fatal(err)
}
output.Line(line, report)
}
func claudeUsageFixture(input, output, cacheRead, cacheCreate int) map[string]any {
return map[string]any{
"input_tokens": input, "output_tokens": output,
"cache_read_input_tokens": cacheRead, "cache_creation_input_tokens": cacheCreate,
}
}