fix(archied): omit the max-token bound on reasoning-class models
The OpenAI chat-completions provider in the pinned ai-sdk serialises a
non-zero output-token bound as `max_tokens`, which reasoning-class models
reject with HTTP 400 ("Use 'max_completion_tokens' instead"). Send no bound
at all for a model the catalog marks reasoning; classic models keep their
bound. Covers the chat turn path and MCP sampling, the two call sites that
set one today.
The reasoning flag rides TurnPrepareContext from the catalog details the
gateway already resolves, so archie-core keeps one notion of "reasoning
model": messaging.ModelDetails.Reasoning. A max_completion_tokens path in
ai-sdk is still needed to ever bound these models; that is filed separately.
This commit is contained in:
1 parent
14c4de77f4
commit
148c81af36
5 files changed
+242
-9
No files matched your search
@@ -60,6 +60,7 @@ func (m *chatTurnModel) Prepare(
|
||||
llm: m.llm(),
|
||||
model: req.Model,
|
||||
options: options,
|
||||
reasoning: req.Reasoning,
|
||||
toolInfo: toolSummaries(options.Tools),
|
||||
toolTokens: gateway.EstimateTokens(string(toolSchema)),
|
||||
outcomes: m.outcomes,
|
||||
@@ -68,9 +69,13 @@ func (m *chatTurnModel) Prepare(
|
||||
}
|
||||
|
||||
type preparedChatTurnModel struct {
|
||||
llm *runtime.Runtime
|
||||
model string
|
||||
options core.GenerateOptions
|
||||
llm *runtime.Runtime
|
||||
model string
|
||||
options core.GenerateOptions
|
||||
// reasoning is the catalog's class for this model, carried from the
|
||||
// prepare context so Generate can choose the output-token parameter the
|
||||
// provider accepts.
|
||||
reasoning bool
|
||||
toolInfo []gateway.ToolSummary
|
||||
toolTokens int
|
||||
outcomes *providerOutcomeRecorder
|
||||
@@ -178,6 +183,20 @@ func (m *preparedChatTurnModel) ToolSchemaTokens() int {
|
||||
return m.toolTokens
|
||||
}
|
||||
|
||||
// maxTokensForRequest returns the output-token bound to place on an ai-sdk
|
||||
// generate request for a model. A reasoning-class model gets no bound at all:
|
||||
// the OpenAI chat-completions provider serialises a non-zero bound as
|
||||
// `max_tokens`, which reasoning models reject with HTTP 400 ("Use
|
||||
// 'max_completion_tokens' instead"), and the pinned ai-sdk release has no
|
||||
// max_completion_tokens path, so no bound is the only usable form. A classic
|
||||
// model keeps the caller's bound unchanged.
|
||||
func maxTokensForRequest(reasoning bool, bound int) int {
|
||||
if reasoning {
|
||||
return 0
|
||||
}
|
||||
return bound
|
||||
}
|
||||
|
||||
func (m *preparedChatTurnModel) Generate(
|
||||
ctx context.Context,
|
||||
request gateway.TurnModelRequest,
|
||||
@@ -187,7 +206,7 @@ func (m *preparedChatTurnModel) Generate(
|
||||
options.Messages = buildTurnMessages(request)
|
||||
// This is one provider response's output allowance, not a turn-
|
||||
// continuation budget. Tool loops remain free to continue.
|
||||
options.MaxTokens = request.MaxOutputTokens
|
||||
options.MaxTokens = maxTokensForRequest(m.reasoning, request.MaxOutputTokens)
|
||||
// A runtime swapped to nil -- a live update that left no usable provider
|
||||
// configured -- is a refused turn, not a panic.
|
||||
if m.llm == nil {
|
||||
|
||||
@@ -95,14 +95,17 @@ func samplingMessages(req mcp.SamplingRequest) ([]chat.Message, error) {
|
||||
|
||||
// samplingMaxTokens bounds the completion to the request's own maxTokens,
|
||||
// falling back to a conservative default, and never past the model's own
|
||||
// output ceiling.
|
||||
// output ceiling. A reasoning-class model gets no bound at all: its provider
|
||||
// rejects the `max_tokens` parameter the chat-completions provider emits for a
|
||||
// non-zero bound.
|
||||
func samplingMaxTokens(req mcp.SamplingRequest, models *chatModelManager, model string) int {
|
||||
details, ok := models.ModelDetails(model)
|
||||
maxTokens := req.MaxTokens
|
||||
if maxTokens <= 0 {
|
||||
maxTokens = samplingDefaultMaxTokens
|
||||
}
|
||||
if details, ok := models.ModelDetails(model); ok && details.MaxOutputTokens > 0 && maxTokens > details.MaxOutputTokens {
|
||||
return details.MaxOutputTokens
|
||||
if ok && details.MaxOutputTokens > 0 && maxTokens > details.MaxOutputTokens {
|
||||
maxTokens = details.MaxOutputTokens
|
||||
}
|
||||
return maxTokens
|
||||
return maxTokensForRequest(ok && details.Reasoning, maxTokens)
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
package archied
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/samcharles93/ai-sdk/runtime"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/agentexec"
|
||||
"github.com/samcharles93/archie-core/internal/gateway"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/modelcatalog"
|
||||
"github.com/samcharles93/archie-core/internal/tools"
|
||||
"github.com/samcharles93/archie-core/internal/tools/mcp"
|
||||
)
|
||||
|
||||
// capturedChatRequest is the JSON body the OpenAI-compatible provider sent for
|
||||
// the last call, which is where the max-token parameter name is chosen.
|
||||
type capturedChatRequest struct {
|
||||
mu sync.Mutex
|
||||
body map[string]any
|
||||
}
|
||||
|
||||
func (c *capturedChatRequest) get() map[string]any {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.body
|
||||
}
|
||||
|
||||
// newCapturingChatAPI answers every completion and records the wire body, so a
|
||||
// test can assert the parameter the provider actually emitted rather than an
|
||||
// option the composition root merely set.
|
||||
func newCapturingChatAPI(t *testing.T, capture *capturedChatRequest) *httptest.Server {
|
||||
t.Helper()
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var body map[string]any
|
||||
_ = json.NewDecoder(r.Body).Decode(&body)
|
||||
capture.mu.Lock()
|
||||
capture.body = body
|
||||
capture.mu.Unlock()
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = io.WriteString(w, `{"id":"chatcmpl-1","object":"chat.completion","created":1,"model":"test",`+
|
||||
`"choices":[{"index":0,"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}],`+
|
||||
`"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}`)
|
||||
}))
|
||||
t.Cleanup(api.Close)
|
||||
return api
|
||||
}
|
||||
|
||||
// assertWireBound asserts the max-token parameter the provider emitted: absent
|
||||
// entirely for a reasoning model (its only accepted form with this ai-sdk
|
||||
// release), present with the caller's bound for a classic one.
|
||||
func assertWireBound(t *testing.T, capture *capturedChatRequest, wantBound bool, wantValue int) {
|
||||
t.Helper()
|
||||
bound, present := capture.get()["max_tokens"]
|
||||
if present != wantBound {
|
||||
t.Fatalf("request body max_tokens present = %v (value %v), want %v; body = %v", present, bound, wantBound, capture.get())
|
||||
}
|
||||
if present && bound != float64(wantValue) {
|
||||
t.Fatalf("request body max_tokens = %v, want %d; body = %v", bound, wantValue, capture.get())
|
||||
}
|
||||
}
|
||||
|
||||
// TestChatTurnParameterNameFollowsModelClass proves the chat turn path reaches
|
||||
// the provider without a max-token bound for a reasoning-class model, while a
|
||||
// classic model still carries one. The bound itself is unchanged; only the
|
||||
// parameter the chat-completions provider would serialise it into differs, and
|
||||
// reasoning models reject `max_tokens` with HTTP 400.
|
||||
func TestChatTurnParameterNameFollowsModelClass(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
reasoning bool
|
||||
wantBound bool
|
||||
modelRef string
|
||||
maxOutput int
|
||||
wantValue int
|
||||
}{
|
||||
{name: "reasoning model", reasoning: true, wantBound: false, modelRef: "openai/gpt-5.4", maxOutput: 4096},
|
||||
{name: "classic model", reasoning: false, wantBound: true, modelRef: "openai/gpt-4o", maxOutput: 4096, wantValue: 4096},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
capture := &capturedChatRequest{}
|
||||
api := newCapturingChatAPI(t, capture)
|
||||
llm := agentexec.NewRuntime(map[string]agentexec.Provider{
|
||||
"openai": {Class: "openai", BaseURL: api.URL},
|
||||
})
|
||||
model := newChatTurnModel(func() *runtime.Runtime { return llm }, tools.NewRegistry(), 1, agentexec.ToolLimits{}, nil)
|
||||
|
||||
prepared, err := model.Prepare(t.Context(), gateway.TurnPrepareContext{
|
||||
Model: tc.modelRef, Reasoning: tc.reasoning,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Prepare() error = %v", err)
|
||||
}
|
||||
_, err = prepared.Generate(t.Context(), gateway.TurnModelRequest{
|
||||
Messages: []gateway.CompressedMessage{{Role: "user", Content: "hi"}},
|
||||
MaxOutputTokens: tc.maxOutput,
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Generate() error = %v", err)
|
||||
}
|
||||
|
||||
assertWireBound(t, capture, tc.wantBound, tc.wantValue)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestMCPSamplingParameterNameFollowsModelClass proves the second place the
|
||||
// process sets a max-token bound -- server-requested sampling, which also
|
||||
// applies a default when the request is silent -- preserves it for a classic
|
||||
// model and drops it entirely for a reasoning one.
|
||||
func TestMCPSamplingParameterNameFollowsModelClass(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
modelRef string
|
||||
model modelcatalog.Model
|
||||
wantBound bool
|
||||
wantValue int
|
||||
}{
|
||||
{
|
||||
name: "reasoning model",
|
||||
modelRef: "openai/gpt-5.4",
|
||||
model: modelcatalog.Model{ID: "gpt-5.4", Reasoning: true, MaxOutputTokens: 128000},
|
||||
},
|
||||
{
|
||||
name: "classic model",
|
||||
modelRef: "openai/gpt-4o",
|
||||
model: modelcatalog.Model{ID: "gpt-4o", MaxOutputTokens: 128000},
|
||||
wantBound: true,
|
||||
wantValue: samplingDefaultMaxTokens,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
capture := &capturedChatRequest{}
|
||||
api := newCapturingChatAPI(t, capture)
|
||||
b := newBootstrap()
|
||||
b.setLLM(agentexec.NewRuntime(map[string]agentexec.Provider{
|
||||
"openai": {Class: "openai", BaseURL: api.URL},
|
||||
}))
|
||||
b.chatModels = newChatModelManager(map[string]string{"chat": tc.modelRef})
|
||||
b.chatModels.SetModelCatalog(modelcatalog.Snapshot{
|
||||
Providers: []modelcatalog.Provider{{ID: "openai", Models: []modelcatalog.Model{tc.model}}},
|
||||
}, []string{tc.modelRef})
|
||||
b.providerOutcomes = newProviderOutcomeRecorder()
|
||||
|
||||
_, err := b.mcpSamplingHandler()(t.Context(), mcp.SamplingRequest{
|
||||
Messages: []mcp.SamplingMessage{{Role: "user", Content: mcp.SamplingContent{Type: "text", Text: "hi"}}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("sampling handler error = %v", err)
|
||||
}
|
||||
|
||||
assertWireBound(t, capture, tc.wantBound, tc.wantValue)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -16,11 +16,18 @@ import (
|
||||
|
||||
// TurnPrepareContext carries the per-turn inputs the model seam needs to build
|
||||
// a generation plan: the active model, the per-turn extra (identity-bound)
|
||||
// tools, and the model's context window used for progressive tool disclosure.
|
||||
// tools, the model's context window used for progressive tool disclosure, and
|
||||
// whether the catalog marks the model reasoning-class, which the provider seam
|
||||
// needs to choose the output-token parameter.
|
||||
type TurnPrepareContext struct {
|
||||
Model string
|
||||
Extra []tools.ToolEntry
|
||||
ContextWindow int
|
||||
// Reasoning reports the catalog's class for the active model. It rides
|
||||
// the prepare context because the seam that translates a provider-neutral
|
||||
// request into provider options is the only place that knows the wire
|
||||
// parameter differs by class.
|
||||
Reasoning bool
|
||||
}
|
||||
|
||||
// TurnModel prepares provider-specific tools and executes one completed chat
|
||||
@@ -333,6 +340,7 @@ func (r *TurnRunner) prepareTurn(ctx context.Context, sessionID string, in Inbou
|
||||
Model: modelName,
|
||||
Extra: extraTools,
|
||||
ContextWindow: modelDetails.ContextWindow,
|
||||
Reasoning: modelDetails.Reasoning,
|
||||
})
|
||||
if err != nil {
|
||||
return preparedTurn{}, fmt.Errorf("build chat model: %w", err)
|
||||
|
||||
@@ -17,9 +17,13 @@ type turnTestModel struct {
|
||||
prepared *turnTestPreparedModel
|
||||
prepareErr error
|
||||
gotExtra []tools.ToolEntry
|
||||
// gotPrepare keeps the whole prepare context so a test can assert what
|
||||
// model metadata the gateway delivered to the seam.
|
||||
gotPrepare TurnPrepareContext
|
||||
}
|
||||
|
||||
func (m *turnTestModel) Prepare(_ context.Context, req TurnPrepareContext) (PreparedTurnModel, error) {
|
||||
m.gotPrepare = req
|
||||
m.gotExtra = append([]tools.ToolEntry(nil), req.Extra...)
|
||||
if m.prepareErr != nil {
|
||||
return nil, m.prepareErr
|
||||
@@ -356,3 +360,39 @@ func TestTurnRunnerReplaysCompletedDuplicateWithoutGenerating(t *testing.T) {
|
||||
t.Fatalf("history length after duplicate = %d, want 2", len(history))
|
||||
}
|
||||
}
|
||||
|
||||
// TestTurnRunnerDeliversModelReasoningClassToTheSeam proves the model class
|
||||
// the provider seam reads comes from the catalog the gateway already resolved,
|
||||
// not from a value the seam invented: a reasoning model's details must reach
|
||||
// Prepare so only the translation layer has to know the wire parameter
|
||||
// differs.
|
||||
func TestTurnRunnerDeliversModelReasoningClassToTheSeam(t *testing.T) {
|
||||
store := NewSessionStoreMemory()
|
||||
router := NewRouter(nil, nil, "telegram")
|
||||
router.Identity = "archie"
|
||||
router.InitSessions(store)
|
||||
|
||||
model := &turnTestModel{prepared: &turnTestPreparedModel{reply: "ok"}}
|
||||
runner := NewTurnRunner(TurnRunnerConfig{
|
||||
Router: router,
|
||||
Sessions: store,
|
||||
Models: &compressTriggerModelManager{
|
||||
models: []string{"openai/gpt-5.4"}, activeModel: "openai/gpt-5.4",
|
||||
details: map[string]ModelDetails{
|
||||
"openai/gpt-5.4": {Ref: "openai/gpt-5.4", ContextWindow: 128000, MaxOutputTokens: 8192, Reasoning: true},
|
||||
},
|
||||
},
|
||||
Personas: NewPersonaRegistry(DefaultPersonas()),
|
||||
Model: model,
|
||||
BotUser: "archie",
|
||||
Channel: "telegram",
|
||||
})
|
||||
|
||||
_, err := runner.Run(context.Background(), Inbound{Message: messaging.Message{SourceID: "source-1", ConversationID: messaging.ConversationID{ChannelID: "chat-1"}, Sender: "user", Role: messaging.RoleUser, Text: "hello"}}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if !model.gotPrepare.Reasoning {
|
||||
t.Fatalf("Prepare reasoning = false, want the catalog's reasoning class to reach the seam")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user