From 148c81af3636b231bcacfe63bf101f99d9ca3d56 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Fri, 2 Oct 2026 10:22:56 +1000 Subject: [PATCH] fix(archied): omit the max-token bound on reasoning-class models The OpenAI chat-completions provider in the pinned ai-sdk serialises a non-zero output-token bound as `max_tokens`, which reasoning-class models reject with HTTP 400 ("Use 'max_completion_tokens' instead"). Send no bound at all for a model the catalog marks reasoning; classic models keep their bound. Covers the chat turn path and MCP sampling, the two call sites that set one today. The reasoning flag rides TurnPrepareContext from the catalog details the gateway already resolves, so archie-core keeps one notion of "reasoning model": messaging.ModelDetails.Reasoning. A max_completion_tokens path in ai-sdk is still needed to ever bound these models; that is filed separately. --- internal/app/archied/chat_turn_model.go | 27 ++- internal/app/archied/mcp_sampling.go | 11 +- .../app/archied/model_output_bound_test.go | 163 ++++++++++++++++++ internal/gateway/turn.go | 10 +- internal/gateway/turn_test.go | 40 +++++ 5 files changed, 242 insertions(+), 9 deletions(-) create mode 100644 internal/app/archied/model_output_bound_test.go diff --git a/internal/app/archied/chat_turn_model.go b/internal/app/archied/chat_turn_model.go index a341f9f4a..f0f9dfb3b 100644 --- a/internal/app/archied/chat_turn_model.go +++ b/internal/app/archied/chat_turn_model.go @@ -60,6 +60,7 @@ func (m *chatTurnModel) Prepare( llm: m.llm(), model: req.Model, options: options, + reasoning: req.Reasoning, toolInfo: toolSummaries(options.Tools), toolTokens: gateway.EstimateTokens(string(toolSchema)), outcomes: m.outcomes, @@ -68,9 +69,13 @@ func (m *chatTurnModel) Prepare( } type preparedChatTurnModel struct { - llm *runtime.Runtime - model string - options core.GenerateOptions + llm *runtime.Runtime + model string + options core.GenerateOptions + // reasoning is the catalog's class for this model, carried from the + // prepare context so Generate can choose the output-token parameter the + // provider accepts. + reasoning bool toolInfo []gateway.ToolSummary toolTokens int outcomes *providerOutcomeRecorder @@ -178,6 +183,20 @@ func (m *preparedChatTurnModel) ToolSchemaTokens() int { return m.toolTokens } +// maxTokensForRequest returns the output-token bound to place on an ai-sdk +// generate request for a model. A reasoning-class model gets no bound at all: +// the OpenAI chat-completions provider serialises a non-zero bound as +// `max_tokens`, which reasoning models reject with HTTP 400 ("Use +// 'max_completion_tokens' instead"), and the pinned ai-sdk release has no +// max_completion_tokens path, so no bound is the only usable form. A classic +// model keeps the caller's bound unchanged. +func maxTokensForRequest(reasoning bool, bound int) int { + if reasoning { + return 0 + } + return bound +} + func (m *preparedChatTurnModel) Generate( ctx context.Context, request gateway.TurnModelRequest, @@ -187,7 +206,7 @@ func (m *preparedChatTurnModel) Generate( options.Messages = buildTurnMessages(request) // This is one provider response's output allowance, not a turn- // continuation budget. Tool loops remain free to continue. - options.MaxTokens = request.MaxOutputTokens + options.MaxTokens = maxTokensForRequest(m.reasoning, request.MaxOutputTokens) // A runtime swapped to nil -- a live update that left no usable provider // configured -- is a refused turn, not a panic. if m.llm == nil { diff --git a/internal/app/archied/mcp_sampling.go b/internal/app/archied/mcp_sampling.go index 52ba499f3..bf8e2e38d 100644 --- a/internal/app/archied/mcp_sampling.go +++ b/internal/app/archied/mcp_sampling.go @@ -95,14 +95,17 @@ func samplingMessages(req mcp.SamplingRequest) ([]chat.Message, error) { // samplingMaxTokens bounds the completion to the request's own maxTokens, // falling back to a conservative default, and never past the model's own -// output ceiling. +// output ceiling. A reasoning-class model gets no bound at all: its provider +// rejects the `max_tokens` parameter the chat-completions provider emits for a +// non-zero bound. func samplingMaxTokens(req mcp.SamplingRequest, models *chatModelManager, model string) int { + details, ok := models.ModelDetails(model) maxTokens := req.MaxTokens if maxTokens <= 0 { maxTokens = samplingDefaultMaxTokens } - if details, ok := models.ModelDetails(model); ok && details.MaxOutputTokens > 0 && maxTokens > details.MaxOutputTokens { - return details.MaxOutputTokens + if ok && details.MaxOutputTokens > 0 && maxTokens > details.MaxOutputTokens { + maxTokens = details.MaxOutputTokens } - return maxTokens + return maxTokensForRequest(ok && details.Reasoning, maxTokens) } diff --git a/internal/app/archied/model_output_bound_test.go b/internal/app/archied/model_output_bound_test.go new file mode 100644 index 000000000..c51bc4a76 --- /dev/null +++ b/internal/app/archied/model_output_bound_test.go @@ -0,0 +1,163 @@ +package archied + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "sync" + "testing" + + "github.com/samcharles93/ai-sdk/runtime" + + "github.com/samcharles93/archie-core/internal/agentexec" + "github.com/samcharles93/archie-core/internal/gateway" + "github.com/samcharles93/archie-core/internal/infrastructure/modelcatalog" + "github.com/samcharles93/archie-core/internal/tools" + "github.com/samcharles93/archie-core/internal/tools/mcp" +) + +// capturedChatRequest is the JSON body the OpenAI-compatible provider sent for +// the last call, which is where the max-token parameter name is chosen. +type capturedChatRequest struct { + mu sync.Mutex + body map[string]any +} + +func (c *capturedChatRequest) get() map[string]any { + c.mu.Lock() + defer c.mu.Unlock() + return c.body +} + +// newCapturingChatAPI answers every completion and records the wire body, so a +// test can assert the parameter the provider actually emitted rather than an +// option the composition root merely set. +func newCapturingChatAPI(t *testing.T, capture *capturedChatRequest) *httptest.Server { + t.Helper() + api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var body map[string]any + _ = json.NewDecoder(r.Body).Decode(&body) + capture.mu.Lock() + capture.body = body + capture.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"id":"chatcmpl-1","object":"chat.completion","created":1,"model":"test",`+ + `"choices":[{"index":0,"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}],`+ + `"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}`) + })) + t.Cleanup(api.Close) + return api +} + +// assertWireBound asserts the max-token parameter the provider emitted: absent +// entirely for a reasoning model (its only accepted form with this ai-sdk +// release), present with the caller's bound for a classic one. +func assertWireBound(t *testing.T, capture *capturedChatRequest, wantBound bool, wantValue int) { + t.Helper() + bound, present := capture.get()["max_tokens"] + if present != wantBound { + t.Fatalf("request body max_tokens present = %v (value %v), want %v; body = %v", present, bound, wantBound, capture.get()) + } + if present && bound != float64(wantValue) { + t.Fatalf("request body max_tokens = %v, want %d; body = %v", bound, wantValue, capture.get()) + } +} + +// TestChatTurnParameterNameFollowsModelClass proves the chat turn path reaches +// the provider without a max-token bound for a reasoning-class model, while a +// classic model still carries one. The bound itself is unchanged; only the +// parameter the chat-completions provider would serialise it into differs, and +// reasoning models reject `max_tokens` with HTTP 400. +func TestChatTurnParameterNameFollowsModelClass(t *testing.T) { + tests := []struct { + name string + reasoning bool + wantBound bool + modelRef string + maxOutput int + wantValue int + }{ + {name: "reasoning model", reasoning: true, wantBound: false, modelRef: "openai/gpt-5.4", maxOutput: 4096}, + {name: "classic model", reasoning: false, wantBound: true, modelRef: "openai/gpt-4o", maxOutput: 4096, wantValue: 4096}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + capture := &capturedChatRequest{} + api := newCapturingChatAPI(t, capture) + llm := agentexec.NewRuntime(map[string]agentexec.Provider{ + "openai": {Class: "openai", BaseURL: api.URL}, + }) + model := newChatTurnModel(func() *runtime.Runtime { return llm }, tools.NewRegistry(), 1, agentexec.ToolLimits{}, nil) + + prepared, err := model.Prepare(t.Context(), gateway.TurnPrepareContext{ + Model: tc.modelRef, Reasoning: tc.reasoning, + }) + if err != nil { + t.Fatalf("Prepare() error = %v", err) + } + _, err = prepared.Generate(t.Context(), gateway.TurnModelRequest{ + Messages: []gateway.CompressedMessage{{Role: "user", Content: "hi"}}, + MaxOutputTokens: tc.maxOutput, + }, nil) + if err != nil { + t.Fatalf("Generate() error = %v", err) + } + + assertWireBound(t, capture, tc.wantBound, tc.wantValue) + }) + } +} + +// TestMCPSamplingParameterNameFollowsModelClass proves the second place the +// process sets a max-token bound -- server-requested sampling, which also +// applies a default when the request is silent -- preserves it for a classic +// model and drops it entirely for a reasoning one. +func TestMCPSamplingParameterNameFollowsModelClass(t *testing.T) { + tests := []struct { + name string + modelRef string + model modelcatalog.Model + wantBound bool + wantValue int + }{ + { + name: "reasoning model", + modelRef: "openai/gpt-5.4", + model: modelcatalog.Model{ID: "gpt-5.4", Reasoning: true, MaxOutputTokens: 128000}, + }, + { + name: "classic model", + modelRef: "openai/gpt-4o", + model: modelcatalog.Model{ID: "gpt-4o", MaxOutputTokens: 128000}, + wantBound: true, + wantValue: samplingDefaultMaxTokens, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + capture := &capturedChatRequest{} + api := newCapturingChatAPI(t, capture) + b := newBootstrap() + b.setLLM(agentexec.NewRuntime(map[string]agentexec.Provider{ + "openai": {Class: "openai", BaseURL: api.URL}, + })) + b.chatModels = newChatModelManager(map[string]string{"chat": tc.modelRef}) + b.chatModels.SetModelCatalog(modelcatalog.Snapshot{ + Providers: []modelcatalog.Provider{{ID: "openai", Models: []modelcatalog.Model{tc.model}}}, + }, []string{tc.modelRef}) + b.providerOutcomes = newProviderOutcomeRecorder() + + _, err := b.mcpSamplingHandler()(t.Context(), mcp.SamplingRequest{ + Messages: []mcp.SamplingMessage{{Role: "user", Content: mcp.SamplingContent{Type: "text", Text: "hi"}}}, + }) + if err != nil { + t.Fatalf("sampling handler error = %v", err) + } + + assertWireBound(t, capture, tc.wantBound, tc.wantValue) + }) + } +} diff --git a/internal/gateway/turn.go b/internal/gateway/turn.go index f6f5c7221..3022c4d6d 100644 --- a/internal/gateway/turn.go +++ b/internal/gateway/turn.go @@ -16,11 +16,18 @@ import ( // TurnPrepareContext carries the per-turn inputs the model seam needs to build // a generation plan: the active model, the per-turn extra (identity-bound) -// tools, and the model's context window used for progressive tool disclosure. +// tools, the model's context window used for progressive tool disclosure, and +// whether the catalog marks the model reasoning-class, which the provider seam +// needs to choose the output-token parameter. type TurnPrepareContext struct { Model string Extra []tools.ToolEntry ContextWindow int + // Reasoning reports the catalog's class for the active model. It rides + // the prepare context because the seam that translates a provider-neutral + // request into provider options is the only place that knows the wire + // parameter differs by class. + Reasoning bool } // TurnModel prepares provider-specific tools and executes one completed chat @@ -333,6 +340,7 @@ func (r *TurnRunner) prepareTurn(ctx context.Context, sessionID string, in Inbou Model: modelName, Extra: extraTools, ContextWindow: modelDetails.ContextWindow, + Reasoning: modelDetails.Reasoning, }) if err != nil { return preparedTurn{}, fmt.Errorf("build chat model: %w", err) diff --git a/internal/gateway/turn_test.go b/internal/gateway/turn_test.go index 0c8e3ef4a..167bcb8c1 100644 --- a/internal/gateway/turn_test.go +++ b/internal/gateway/turn_test.go @@ -17,9 +17,13 @@ type turnTestModel struct { prepared *turnTestPreparedModel prepareErr error gotExtra []tools.ToolEntry + // gotPrepare keeps the whole prepare context so a test can assert what + // model metadata the gateway delivered to the seam. + gotPrepare TurnPrepareContext } func (m *turnTestModel) Prepare(_ context.Context, req TurnPrepareContext) (PreparedTurnModel, error) { + m.gotPrepare = req m.gotExtra = append([]tools.ToolEntry(nil), req.Extra...) if m.prepareErr != nil { return nil, m.prepareErr @@ -356,3 +360,39 @@ func TestTurnRunnerReplaysCompletedDuplicateWithoutGenerating(t *testing.T) { t.Fatalf("history length after duplicate = %d, want 2", len(history)) } } + +// TestTurnRunnerDeliversModelReasoningClassToTheSeam proves the model class +// the provider seam reads comes from the catalog the gateway already resolved, +// not from a value the seam invented: a reasoning model's details must reach +// Prepare so only the translation layer has to know the wire parameter +// differs. +func TestTurnRunnerDeliversModelReasoningClassToTheSeam(t *testing.T) { + store := NewSessionStoreMemory() + router := NewRouter(nil, nil, "telegram") + router.Identity = "archie" + router.InitSessions(store) + + model := &turnTestModel{prepared: &turnTestPreparedModel{reply: "ok"}} + runner := NewTurnRunner(TurnRunnerConfig{ + Router: router, + Sessions: store, + Models: &compressTriggerModelManager{ + models: []string{"openai/gpt-5.4"}, activeModel: "openai/gpt-5.4", + details: map[string]ModelDetails{ + "openai/gpt-5.4": {Ref: "openai/gpt-5.4", ContextWindow: 128000, MaxOutputTokens: 8192, Reasoning: true}, + }, + }, + Personas: NewPersonaRegistry(DefaultPersonas()), + Model: model, + BotUser: "archie", + Channel: "telegram", + }) + + _, err := runner.Run(context.Background(), Inbound{Message: messaging.Message{SourceID: "source-1", ConversationID: messaging.ConversationID{ChannelID: "chat-1"}, Sender: "user", Role: messaging.RoleUser, Text: "hello"}}, nil) + if err != nil { + t.Fatalf("Run() error = %v", err) + } + if !model.gotPrepare.Reasoning { + t.Fatalf("Prepare reasoning = false, want the catalog's reasoning class to reach the seam") + } +}