From 0eef8ed5ba40e435eca88cfc824e514d5ce900e3 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Sat, 26 Sep 2026 03:34:47 +1000 Subject: [PATCH] fix(agentexec): count Claude cache tokens as prompt tokens, as the built-in loop does Claude's input_tokens excludes cache reads and writes; the built-in loop's Anthropic provider folds them into PromptTokens so budgets count total prompt size. The harness adapter now does the same, so one budget means the same on either runner. --- internal/agentexec/harness_output_claude.go | 25 ++++++++----------- .../agentexec/harness_output_claude_test.go | 8 +++--- 2 files changed, 14 insertions(+), 19 deletions(-) diff --git a/internal/agentexec/harness_output_claude.go b/internal/agentexec/harness_output_claude.go index 82bd96f56..ac99313ba 100644 --- a/internal/agentexec/harness_output_claude.go +++ b/internal/agentexec/harness_output_claude.go @@ -55,13 +55,8 @@ func (o *claudeCodeHarnessOutput) Line(line []byte, report ToolCallReporter) { func (o *claudeCodeHarnessOutput) assistant(raw json.RawMessage, _ ToolCallReporter) { var message struct { - ID string `json:"id"` - Usage *struct { - InputTokens int `json:"input_tokens"` - OutputTokens int `json:"output_tokens"` - CacheReadInputTokens int `json:"cache_read_input_tokens"` - CacheCreationInputTokens int `json:"cache_creation_input_tokens"` - } `json:"usage"` + ID string `json:"id"` + Usage json.RawMessage `json:"usage"` Content []struct { Type string `json:"type"` ID string `json:"id"` @@ -72,13 +67,8 @@ func (o *claudeCodeHarnessOutput) assistant(raw json.RawMessage, _ ToolCallRepor if json.Unmarshal(raw, &message) != nil { return } - if message.Usage != nil && (message.ID == "" || o.markMessage(message.ID)) { - o.messageUsage = addUsage(o.messageUsage, usageFromFields( - message.Usage.InputTokens, - message.Usage.OutputTokens, - message.Usage.CacheReadInputTokens, - message.Usage.CacheCreationInputTokens, - )) + if usage, ok := decodeClaudeUsage(message.Usage); ok && (message.ID == "" || o.markMessage(message.ID)) { + o.messageUsage = addUsage(o.messageUsage, usage) } for _, block := range message.Content { if block.Type == "tool_use" && block.ID != "" && block.Name != "" { @@ -147,9 +137,14 @@ func decodeClaudeUsage(raw json.RawMessage) (Usage, bool) { return usageFromFields(fields.InputTokens, fields.OutputTokens, fields.CacheReadInputTokens, fields.CacheCreationInputTokens), true } +// usageFromFields folds cache reads and writes into PromptTokens, as the +// built-in loop's Anthropic provider does: Claude's input_tokens excludes +// them, and budgets must count a harness run the way they count a built-in +// one. func usageFromFields(input, output, cacheRead, cacheCreation int) Usage { + prompt := input + cacheRead + cacheCreation return Usage{ - PromptTokens: input, CompletionTokens: output, TotalTokens: input + output, + PromptTokens: prompt, CompletionTokens: output, TotalTokens: prompt + output, CachedTokens: cacheRead, CacheCreationTokens: cacheCreation, } } diff --git a/internal/agentexec/harness_output_claude_test.go b/internal/agentexec/harness_output_claude_test.go index c5a1f0051..242405a35 100644 --- a/internal/agentexec/harness_output_claude_test.go +++ b/internal/agentexec/harness_output_claude_test.go @@ -23,7 +23,7 @@ func TestClaudeCodeHarnessOutputFixtures(t *testing.T) { {Tool: "Bash", Detail: "hello"}, {Tool: "Bash", Detail: errorDetail, Failed: true}, }, - wantUsage: Usage{PromptTokens: 6, CompletionTokens: 248, TotalTokens: 254, CachedTokens: 78105, CacheCreationTokens: 24309}, + wantUsage: Usage{PromptTokens: 102420, CompletionTokens: 248, TotalTokens: 102668, CachedTokens: 78105, CacheCreationTokens: 24309}, wantSession: "9c22c15c-80ef-42c6-a503-8d087ec47d92", }, { @@ -32,7 +32,7 @@ func TestClaudeCodeHarnessOutputFixtures(t *testing.T) { {Tool: "Bash", Detail: "hello"}, {Tool: "Bash", Detail: errorDetail, Failed: true}, }, - wantUsage: Usage{PromptTokens: 6, CompletionTokens: 121, TotalTokens: 127, CachedTokens: 78105, CacheCreationTokens: 24309}, + wantUsage: Usage{PromptTokens: 102420, CompletionTokens: 121, TotalTokens: 102541, CachedTokens: 78105, CacheCreationTokens: 24309}, wantSession: "9c22c15c-80ef-42c6-a503-8d087ec47d92", }, { @@ -41,7 +41,7 @@ func TestClaudeCodeHarnessOutputFixtures(t *testing.T) { {Tool: "Bash", Detail: "hello"}, {Tool: "Bash", Detail: errorDetail, Failed: true}, }, - wantUsage: Usage{PromptTokens: 6, CompletionTokens: 248, TotalTokens: 254, CachedTokens: 78105, CacheCreationTokens: 24309}, + wantUsage: Usage{PromptTokens: 102420, CompletionTokens: 248, TotalTokens: 102668, CachedTokens: 78105, CacheCreationTokens: 24309}, wantSession: "9c22c15c-80ef-42c6-a503-8d087ec47d92", }, } @@ -120,7 +120,7 @@ func TestClaudeCodeHarnessOutputToolResultContent(t *testing.T) { if !reflect.DeepEqual(calls, wantCalls) { t.Errorf("tool calls = %#v, want %#v", calls, wantCalls) } - wantUsage := Usage{PromptTokens: 2, CompletionTokens: 3, TotalTokens: 5, CachedTokens: 5, CacheCreationTokens: 7} + wantUsage := Usage{PromptTokens: 14, CompletionTokens: 3, TotalTokens: 17, CachedTokens: 5, CacheCreationTokens: 7} if got := output.Usage(); got != wantUsage { t.Errorf("usage = %#v, want %#v", got, wantUsage) }