feat(gateway): stream model reasoning as reasoning chat events (#1907)
The ai-sdk FullStream drain moves from the archiegateway composition into
internal/gateway (DrainStreamParts), next to the TurnStream it feeds, and
gains a StreamPartReasoningDelta case: each reasoning delta is forwarded on
the new optional ReasoningStream method, which localChatStream turns into a
ChatEvent{Kind: "reasoning", Text: delta} on the same stream. done/error
semantics are unchanged, and the FullStream is still drained to close when no
stream is attached.
This commit is contained in:
4 files changed
+292
-121
No files matched your search
@@ -2,13 +2,9 @@ package archiegateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/samcharles93/ai-sdk/chat"
|
||||
"github.com/samcharles93/ai-sdk/core"
|
||||
@@ -220,7 +216,7 @@ func runChatTurn(ctx context.Context, llm *runtime.Runtime, chatModel string, op
|
||||
if err != nil {
|
||||
return "", chat.Usage{}, fmt.Errorf("llm chat stream: %w", err)
|
||||
}
|
||||
text := drainChatStream(stream.FullStream, turn, icons)
|
||||
text := gateway.DrainStreamParts(stream.FullStream, turn, icons)
|
||||
var used chat.Usage
|
||||
if stream.Usage != nil {
|
||||
used, _ = stream.Usage()
|
||||
@@ -231,121 +227,6 @@ func runChatTurn(ctx context.Context, llm *runtime.Runtime, chatModel string, op
|
||||
return text, used, nil
|
||||
}
|
||||
|
||||
// pendingToolCall is a tool invocation awaiting its result.
|
||||
type pendingToolCall struct {
|
||||
name string
|
||||
parameters string
|
||||
started time.Time
|
||||
offset int
|
||||
}
|
||||
|
||||
// took is the call's running time so far, zero for a result with no
|
||||
// recorded invocation.
|
||||
func (c pendingToolCall) took() int64 {
|
||||
if c.started.IsZero() {
|
||||
return 0
|
||||
}
|
||||
return time.Since(c.started).Milliseconds()
|
||||
}
|
||||
|
||||
// drainChatStream consumes a model stream to close, reporting assistant text
|
||||
// and each completed tool call to turn, and returns the assembled reply.
|
||||
//
|
||||
// FullStream is authoritative and its writes are synchronous, so it must be
|
||||
// drained to close even when turn is nil -- an unread stream stalls the
|
||||
// generating goroutine.
|
||||
//
|
||||
// A tool is reported on its result, never on its call: the call part carries
|
||||
// no outcome yet, so reporting both would show every tool twice, once with
|
||||
// nothing to say.
|
||||
func drainChatStream(parts <-chan core.StreamPart, turn gateway.TurnStream, icons map[string]string) string {
|
||||
var sb strings.Builder
|
||||
pending := make(map[string]pendingToolCall)
|
||||
runes := 0
|
||||
for part := range parts {
|
||||
switch part.Type {
|
||||
case core.StreamPartTextDelta:
|
||||
if part.TextDelta == "" {
|
||||
continue
|
||||
}
|
||||
sb.WriteString(part.TextDelta)
|
||||
runes += utf8.RuneCountInString(part.TextDelta)
|
||||
if turn != nil {
|
||||
turn.Delta(part.TextDelta)
|
||||
}
|
||||
case core.StreamPartToolCall:
|
||||
if part.ToolCall == nil || part.ToolCall.ToolCallID == "" {
|
||||
continue
|
||||
}
|
||||
pending[part.ToolCall.ToolCallID] = pendingToolCall{
|
||||
name: part.ToolCall.ToolName,
|
||||
parameters: gateway.SummarizeToolParameters(part.ToolCall.Input),
|
||||
started: time.Now(),
|
||||
offset: runes,
|
||||
}
|
||||
case core.StreamPartToolResult:
|
||||
if part.ToolResult == nil || turn == nil {
|
||||
continue
|
||||
}
|
||||
call := pending[part.ToolResult.ToolCallID]
|
||||
delete(pending, part.ToolResult.ToolCallID)
|
||||
name := part.ToolResult.ToolName
|
||||
if name == "" {
|
||||
name = call.name
|
||||
}
|
||||
turn.ToolCall(gateway.ToolCallEvent{
|
||||
ID: part.ToolResult.ToolCallID,
|
||||
Name: name,
|
||||
Emoji: icons[name],
|
||||
Parameters: call.parameters,
|
||||
Output: part.ToolResult.Output,
|
||||
Err: part.ToolResult.Error,
|
||||
DurationMS: call.took(),
|
||||
TextOffset: call.offset,
|
||||
})
|
||||
for _, ref := range multimodalMediaRefs(part.ToolResult.Output) {
|
||||
// Path and URL are alternatives, not a pair: the channel
|
||||
// uploads one and fetches the other. A ref carrying
|
||||
// neither names nothing deliverable, so it is dropped
|
||||
// rather than sent as an empty attachment.
|
||||
if ref.URL == "" && ref.Path == "" {
|
||||
continue
|
||||
}
|
||||
turn.Media(gateway.MediaEvent{
|
||||
ToolName: name,
|
||||
Attachment: gateway.MediaAttachment{
|
||||
Type: ref.Type,
|
||||
URL: ref.URL,
|
||||
Path: ref.Path,
|
||||
FileName: ref.FileName,
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
// multimodalMediaRefs extracts the URLs a tool result carries when its
|
||||
// output is a tools.MultimodalResult, so drainChatStream can report each as
|
||||
// a gateway.MediaEvent alongside the ordinary ToolCall event.
|
||||
//
|
||||
// output is decoded with json.Unmarshal directly rather than checked
|
||||
// against a schema first: most tool output is plain text or unrelated
|
||||
// JSON, and unmarshalling into MultimodalResult simply leaves IsMultimodal
|
||||
// false for anything that doesn't match, which is exactly "not multimodal"
|
||||
// -- no separate detection step earns its keep here.
|
||||
func multimodalMediaRefs(output string) []tools.MediaRef {
|
||||
if output == "" {
|
||||
return nil
|
||||
}
|
||||
var result tools.MultimodalResult
|
||||
if err := json.Unmarshal([]byte(output), &result); err != nil || !result.IsMultimodal {
|
||||
return nil
|
||||
}
|
||||
return result.URLs
|
||||
}
|
||||
|
||||
// chatTitleGenerator proposes session titles through the chat model. It
|
||||
// is a pure proposal: it persists nothing, so a failed or slow title
|
||||
// call can never corrupt conversation history or fail the turn. The
|
||||
|
||||
@@ -331,6 +331,12 @@ func (s localChatStream) Delta(text string) {
|
||||
s.send(ChatEvent{Kind: "delta", Text: text})
|
||||
}
|
||||
|
||||
// Reasoning forwards one fragment of the model's thinking as its own event
|
||||
// kind; the dashboard renders it as the live turn's thinking trace.
|
||||
func (s localChatStream) Reasoning(text string) {
|
||||
s.send(ChatEvent{Kind: "reasoning", Text: text})
|
||||
}
|
||||
|
||||
func (s localChatStream) ToolCall(event ToolCallEvent) {
|
||||
s.send(ChatEvent{Kind: "tool", Tool: event})
|
||||
}
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/samcharles93/ai-sdk/core"
|
||||
)
|
||||
|
||||
// A fake ai-sdk FullStream drives the drain: every part must reach the turn
|
||||
// stream in order, and a reasoning delta must surface as a Kind "reasoning"
|
||||
// ChatEvent carrying its text. Text deltas and tool results keep working.
|
||||
func TestDrainStreamPartsForwardsStreamParts(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
parts []core.StreamPart
|
||||
want []ChatEvent
|
||||
wantText string
|
||||
}{
|
||||
{
|
||||
name: "reasoning deltas become reasoning events",
|
||||
parts: []core.StreamPart{
|
||||
{Type: core.StreamPartReasoningDelta, ReasoningDelta: "considering "},
|
||||
{Type: core.StreamPartReasoningDelta, ReasoningDelta: "the request"},
|
||||
},
|
||||
want: []ChatEvent{
|
||||
{Kind: "reasoning", Text: "considering ", SessionID: "s1"},
|
||||
{Kind: "reasoning", Text: "the request", SessionID: "s1"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "text deltas still stream and assemble the reply",
|
||||
parts: []core.StreamPart{
|
||||
{Type: core.StreamPartTextDelta, TextDelta: "Hello, "},
|
||||
{Type: core.StreamPartTextDelta, TextDelta: "world"},
|
||||
},
|
||||
want: []ChatEvent{
|
||||
{Kind: "delta", Text: "Hello, ", SessionID: "s1"},
|
||||
{Kind: "delta", Text: "world", SessionID: "s1"},
|
||||
},
|
||||
wantText: "Hello, world",
|
||||
},
|
||||
{
|
||||
name: "reasoning ahead of the answer text",
|
||||
parts: []core.StreamPart{
|
||||
{Type: core.StreamPartReasoningDelta, ReasoningDelta: "hmm"},
|
||||
{Type: core.StreamPartTextDelta, TextDelta: "answer"},
|
||||
},
|
||||
want: []ChatEvent{
|
||||
{Kind: "reasoning", Text: "hmm", SessionID: "s1"},
|
||||
{Kind: "delta", Text: "answer", SessionID: "s1"},
|
||||
},
|
||||
wantText: "answer",
|
||||
},
|
||||
{
|
||||
name: "an empty reasoning delta is dropped",
|
||||
parts: []core.StreamPart{
|
||||
{Type: core.StreamPartReasoningDelta, ReasoningDelta: ""},
|
||||
{Type: core.StreamPartTextDelta, TextDelta: "hi"},
|
||||
},
|
||||
want: []ChatEvent{{Kind: "delta", Text: "hi", SessionID: "s1"}},
|
||||
wantText: "hi",
|
||||
},
|
||||
{
|
||||
name: "a completed tool call still reports its tool event",
|
||||
parts: []core.StreamPart{
|
||||
{Type: core.StreamPartToolCall, ToolCall: &core.ToolCall{ToolCallID: "t1", ToolName: "search", Input: `{"q":"archie"}`}},
|
||||
{Type: core.StreamPartToolResult, ToolResult: &core.ToolResult{ToolCallID: "t1", ToolName: "search", Output: "42"}},
|
||||
},
|
||||
want: []ChatEvent{{
|
||||
Kind: "tool",
|
||||
SessionID: "s1",
|
||||
Tool: ToolCallEvent{
|
||||
ID: "t1",
|
||||
Name: "search",
|
||||
Emoji: "🔍",
|
||||
Parameters: SummarizeToolParameters(`{"q":"archie"}`),
|
||||
Output: "42",
|
||||
},
|
||||
}},
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
parts := make(chan core.StreamPart, len(tc.parts))
|
||||
for _, part := range tc.parts {
|
||||
parts <- part
|
||||
}
|
||||
close(parts)
|
||||
|
||||
events := make(chan ChatEvent, len(tc.parts)+1)
|
||||
stream := localChatStream{done: make(chan struct{}), events: events, sessionID: "s1"}
|
||||
text := DrainStreamParts(parts, stream, map[string]string{"search": "🔍"})
|
||||
|
||||
var got []ChatEvent
|
||||
for {
|
||||
select {
|
||||
case event := <-events:
|
||||
got = append(got, event)
|
||||
default:
|
||||
if !reflect.DeepEqual(got, tc.want) {
|
||||
t.Fatalf("events = %+v, want %+v", got, tc.want)
|
||||
}
|
||||
if text != tc.wantText {
|
||||
t.Fatalf("reply text = %q, want %q", text, tc.wantText)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A stream nobody renders still has to be drained to close: an unread
|
||||
// FullStream stalls the generating goroutine, reasoning included.
|
||||
func TestDrainStreamPartsWithoutStreamStillDrains(t *testing.T) {
|
||||
parts := make(chan core.StreamPart, 3)
|
||||
parts <- core.StreamPart{Type: core.StreamPartReasoningDelta, ReasoningDelta: "thinking"}
|
||||
parts <- core.StreamPart{Type: core.StreamPartTextDelta, TextDelta: "reply"}
|
||||
close(parts)
|
||||
if text := DrainStreamParts(parts, nil, nil); text != "reply" {
|
||||
t.Fatalf("reply text = %q, want %q", text, "reply")
|
||||
}
|
||||
}
|
||||
+161
-1
@@ -1,12 +1,30 @@
|
||||
package gateway
|
||||
|
||||
import "github.com/samcharles93/archie-core/internal/domain/messaging"
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/samcharles93/ai-sdk/core"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/domain/messaging"
|
||||
"github.com/samcharles93/archie-core/internal/tools"
|
||||
)
|
||||
|
||||
// SummarizeToolParameters returns a bounded JSON shape of tool input.
|
||||
func SummarizeToolParameters(input string) string {
|
||||
return messaging.SummarizeToolParameters(input)
|
||||
}
|
||||
|
||||
// ReasoningStream is the optional TurnStream extension for the model's
|
||||
// thinking. It is separate from TurnStream so streams that cannot render
|
||||
// reasoning keep satisfying TurnStream unchanged; producers type-assert.
|
||||
type ReasoningStream interface {
|
||||
// Reasoning appends the next fragment of the model's reasoning.
|
||||
Reasoning(text string)
|
||||
}
|
||||
|
||||
type toolCallRecorder struct {
|
||||
next TurnStream
|
||||
events []ToolCallEvent
|
||||
@@ -18,6 +36,12 @@ func (r *toolCallRecorder) Delta(text string) {
|
||||
}
|
||||
}
|
||||
|
||||
func (r *toolCallRecorder) Reasoning(text string) {
|
||||
if rs, ok := r.next.(ReasoningStream); ok {
|
||||
rs.Reasoning(text)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *toolCallRecorder) ToolCall(event ToolCallEvent) {
|
||||
r.events = append(r.events, event)
|
||||
if r.next != nil {
|
||||
@@ -30,3 +54,139 @@ func (r *toolCallRecorder) Media(event MediaEvent) {
|
||||
r.next.Media(event)
|
||||
}
|
||||
}
|
||||
|
||||
// pendingToolCall is a tool invocation awaiting its result.
|
||||
type pendingToolCall struct {
|
||||
name string
|
||||
parameters string
|
||||
started time.Time
|
||||
offset int
|
||||
}
|
||||
|
||||
// took is the call's running time so far, zero for a result with no
|
||||
// recorded invocation.
|
||||
func (c pendingToolCall) took() int64 {
|
||||
if c.started.IsZero() {
|
||||
return 0
|
||||
}
|
||||
return time.Since(c.started).Milliseconds()
|
||||
}
|
||||
|
||||
// DrainStreamParts consumes an ai-sdk FullStream to close, reporting the
|
||||
// model's reasoning, assistant text and each completed tool call to turn,
|
||||
// and returns the assembled reply.
|
||||
//
|
||||
// The parts channel is authoritative and its writes are synchronous, so it
|
||||
// must be drained to close even when turn is nil -- an unread stream stalls
|
||||
// the generating goroutine.
|
||||
//
|
||||
// A tool is reported on its result, never on its call: the call part carries
|
||||
// no outcome yet, so reporting both would show every tool twice, once with
|
||||
// nothing to say.
|
||||
func DrainStreamParts(parts <-chan core.StreamPart, turn TurnStream, icons map[string]string) string {
|
||||
var sb strings.Builder
|
||||
pending := make(map[string]pendingToolCall)
|
||||
runes := 0
|
||||
for part := range parts {
|
||||
switch part.Type {
|
||||
case core.StreamPartReasoningDelta:
|
||||
reportReasoning(turn, part.ReasoningDelta)
|
||||
case core.StreamPartTextDelta:
|
||||
if part.TextDelta == "" {
|
||||
continue
|
||||
}
|
||||
sb.WriteString(part.TextDelta)
|
||||
runes += utf8.RuneCountInString(part.TextDelta)
|
||||
if turn != nil {
|
||||
turn.Delta(part.TextDelta)
|
||||
}
|
||||
case core.StreamPartToolCall:
|
||||
if part.ToolCall == nil || part.ToolCall.ToolCallID == "" {
|
||||
continue
|
||||
}
|
||||
pending[part.ToolCall.ToolCallID] = pendingToolCall{
|
||||
name: part.ToolCall.ToolName,
|
||||
parameters: SummarizeToolParameters(part.ToolCall.Input),
|
||||
started: time.Now(),
|
||||
offset: runes,
|
||||
}
|
||||
case core.StreamPartToolResult:
|
||||
reportToolResult(turn, part.ToolResult, pending, icons)
|
||||
}
|
||||
}
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
// reportReasoning forwards one reasoning fragment to a stream that can
|
||||
// render thinking. Reasoning is not part of the reply text, so it travels on
|
||||
// its own stream method; a stream without that method simply gets none.
|
||||
func reportReasoning(turn TurnStream, delta string) {
|
||||
if delta == "" {
|
||||
return
|
||||
}
|
||||
if rs, ok := turn.(ReasoningStream); ok {
|
||||
rs.Reasoning(delta)
|
||||
}
|
||||
}
|
||||
|
||||
// reportToolResult reports one finished tool invocation, plus any media
|
||||
// attachment its output carries, to the turn.
|
||||
func reportToolResult(turn TurnStream, result *core.ToolResult, pending map[string]pendingToolCall, icons map[string]string) {
|
||||
if result == nil || turn == nil {
|
||||
return
|
||||
}
|
||||
call := pending[result.ToolCallID]
|
||||
delete(pending, result.ToolCallID)
|
||||
name := result.ToolName
|
||||
if name == "" {
|
||||
name = call.name
|
||||
}
|
||||
turn.ToolCall(ToolCallEvent{
|
||||
ID: result.ToolCallID,
|
||||
Name: name,
|
||||
Emoji: icons[name],
|
||||
Parameters: call.parameters,
|
||||
Output: result.Output,
|
||||
Err: result.Error,
|
||||
DurationMS: call.took(),
|
||||
TextOffset: call.offset,
|
||||
})
|
||||
for _, ref := range multimodalMediaRefs(result.Output) {
|
||||
// Path and URL are alternatives, not a pair: the channel uploads one
|
||||
// and fetches the other. A ref carrying neither names nothing
|
||||
// deliverable, so it is dropped rather than sent as an empty
|
||||
// attachment.
|
||||
if ref.URL == "" && ref.Path == "" {
|
||||
continue
|
||||
}
|
||||
turn.Media(MediaEvent{
|
||||
ToolName: name,
|
||||
Attachment: MediaAttachment{
|
||||
Type: ref.Type,
|
||||
URL: ref.URL,
|
||||
Path: ref.Path,
|
||||
FileName: ref.FileName,
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// multimodalMediaRefs extracts the URLs a tool result carries when its
|
||||
// output is a tools.MultimodalResult, so DrainStreamParts can report each as
|
||||
// a MediaEvent alongside the ordinary ToolCall event.
|
||||
//
|
||||
// output is decoded with json.Unmarshal directly rather than checked
|
||||
// against a schema first: most tool output is plain text or unrelated
|
||||
// JSON, and unmarshalling into MultimodalResult simply leaves IsMultimodal
|
||||
// false for anything that doesn't match, which is exactly "not multimodal"
|
||||
// -- no separate detection step earns its keep here.
|
||||
func multimodalMediaRefs(output string) []tools.MediaRef {
|
||||
if output == "" {
|
||||
return nil
|
||||
}
|
||||
var result tools.MultimodalResult
|
||||
if err := json.Unmarshal([]byte(output), &result); err != nil || !result.IsMultimodal {
|
||||
return nil
|
||||
}
|
||||
return result.URLs
|
||||
}
|
||||
Reference in new issue
Block a user