diff --git a/internal/gateway/chat_local_test.go b/internal/gateway/chat_local_test.go index c9dbe288c..c93bf1540 100644 --- a/internal/gateway/chat_local_test.go +++ b/internal/gateway/chat_local_test.go @@ -5,6 +5,39 @@ import ( "testing" ) +// streamChat builds the local chat adapter the way the composition wires +// it, pointed at the router under test; it is the production streaming +// entry these streamed-turn tests drive. +func streamChat(r *Router) *LocalChatAdapter { + return &LocalChatAdapter{Router: r} +} + +// drainStreamed runs one streamed turn through the production chat adapter +// and returns what its stream rendered: the deltas, then the terminal +// event's kind and text. +func drainStreamed(t *testing.T, chat *LocalChatAdapter, in Inbound) ([]string, string, string, error) { + t.Helper() + events, err := chat.Stream(context.Background(), in) + if err != nil { + return nil, "", "", err + } + rendered, drainErr := drained(events) + if drainErr != nil { + return nil, "", "", drainErr + } + var deltas []string + kind, text := "", "" + for _, event := range rendered { + switch event.Kind { + case "delta": + deltas = append(deltas, event.Text) + case "done", "error": + kind, text = event.Kind, event.Text + } + } + return deltas, kind, text, nil +} + func TestLocalChatSnapshotAndStream(t *testing.T) { ctx := t.Context() sessions := NewSessionStoreMemory() diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go index 2cd22f48a..45accd171 100644 --- a/internal/gateway/gateway.go +++ b/internal/gateway/gateway.go @@ -167,9 +167,9 @@ type Router struct { // reporting a value it does not have. Health HealthSource LLM LLMResponder // nil = LLM not wired yet - // LLMStream is the optional streaming responder. When set, adapters - // that can render partial output (see RouteStream) show the reply as - // it generates; when nil, everything falls back to LLM. + // LLMStream is the optional streaming responder. When set, the streamed + // chat adapter (internal/gateway/chat_local.go) shows the reply as it + // generates; when nil, everything falls back to LLM. LLMStream LLMStreamResponder // Identity is the archie identity this router belongs to (empty in // single-identity deployments). Propagated into SpawnRequest and @@ -345,8 +345,9 @@ func (r *Router) CollectTurn(ctx context.Context, in Inbound) (Inbound, bool, er } // route is Route's continuation once the caller has already cleared the -// rate-limit check (or is deliberately skipping it, as RouteStream's -// local-command fallthrough does to avoid charging one message twice). +// rate-limit check (or is deliberately skipping it, as streamTurn's +// local-command fallthrough and the streamed adapter's degraded-to-non- +// streaming path do, to avoid charging one message twice). func (r *Router) route(ctx context.Context, in Inbound) (string, error) { text := strings.TrimSpace(in.Message.Text) cmd, _ := parseCmd(text, r.gatewayName) @@ -502,56 +503,25 @@ func (r *Router) dispatchSessionCommand(ctx context.Context, msg messaging.Messa return "", false, nil } -// RouteStream is Route for adapters that can render a reply progressively. -// stream receives each new fragment of an LLM reply as it is generated, plus -// each completed tool call, in the order the model produced them; the -// complete reply is returned as usual. -// -// Only free-text messages stream. Gateway-local commands (/status, /model, -// …) answer from local state in a single step, so they return through the -// normal path with no delta callbacks. When no streaming responder is -// configured, RouteStream is exactly Route -- callers get the whole reply -// at the end and simply never see a delta, so an adapter can always call -// RouteStream without checking first. -func (r *Router) RouteStream(ctx context.Context, in Inbound, stream TurnStream) (string, error) { - if r.LLMStream == nil || stream == nil { - return r.Route(ctx, in) - } - if r.duplicateDelivery(in) { - return dedupReply, nil - } - if r.checkRateLimit(in) { - return rateLimitReply, nil - } - return r.streamTurn(ctx, in, stream) -} - -// streamTurn dispatches one streaming turn for an inbound whose per-message -// gates have already run. RouteStream splits it off so the text batcher's -// dispatcher can carry the joined payload for its whole batch through here -// without the gates answering a second time for a message they already saw -// -- in particular so the combined payload's first-fragment source ID, which -// the dedup gate already recorded, does not read back as a redelivery. +// streamTurn dispatches one streamed turn for an inbound the streamed chat +// adapter has already prepared: the per-message gates (dedup, rate limit) +// ran per fragment in the adapter's prelude, and the text batcher has +// settled the turn's payload -- prelude's dispatcher carries the joined +// payload for its whole batch here, so the gates answer no second time for +// messages they already saw, and in particular the combined payload's +// first-fragment source ID, which the dedup gate already recorded, does +// not read back as a redelivery. func (r *Router) streamTurn(ctx context.Context, in Inbound, stream TurnStream) (string, error) { + if r.LLMStream == nil || stream == nil { + return r.route(ctx, in) + } cmd, _ := parseCmd(strings.TrimSpace(in.Message.Text), r.gatewayName) if isLocalCommand(cmd) || strings.HasPrefix(cmd, "/") { return r.route(ctx, in) } - ready, mine, err := r.CollectTurn(ctx, in) - if err != nil { - return "", err - } - if !mine { - // A covered fragment: the batch's one turn streams on the call that - // dispatched it, so this caller's stream renders nothing. The empty - // final reply is what every renderer drops (Telegram's finalizer has - // no message and abandons; a dashboard client shows the turn that - // actually ran). - return "", nil - } - reply, err := r.LLMStream(ctx, ready, stream) + reply, err := r.LLMStream(ctx, in, stream) if err == nil { - r.maybeAutoTitle(ctx, ready.Message) + r.maybeAutoTitle(ctx, in.Message) } return reply, err } diff --git a/internal/gateway/gateway_ratelimit_test.go b/internal/gateway/gateway_ratelimit_test.go index c0e1e33b2..25b93f509 100644 --- a/internal/gateway/gateway_ratelimit_test.go +++ b/internal/gateway/gateway_ratelimit_test.go @@ -140,11 +140,11 @@ func TestRouteBudgetKeyLimitsWithoutSenderID(t *testing.T) { } } -// TestRouteStreamLocalCommandChargesExactlyOnce pins RouteStream's local -// command fallthrough (into route, not Route) against double-charging the -// budget: a status query must consume the same one unit whether it goes -// through Route or RouteStream. -func TestRouteStreamLocalCommandChargesExactlyOnce(t *testing.T) { +// TestStreamLocalCommandChargesExactlyOnce pins the streamed adapter's local +// command fallthrough (into route, not the streaming responder) against +// double-charging the budget: a status query must consume the same one unit +// whether it goes through Route or the adapter's Stream. +func TestStreamLocalCommandChargesExactlyOnce(t *testing.T) { r := NewRouter(&fakeStore{counts: map[string]int{}}, nil, "test") r.Limiter = ratelimit.New(time.Minute, 2) // The batcher is off: the charge being pinned is per message, not per batch. @@ -152,31 +152,30 @@ func TestRouteStreamLocalCommandChargesExactlyOnce(t *testing.T) { r.LLMStream = func(context.Context, Inbound, TurnStream) (string, error) { return "streamed", nil } - - if _, err := r.RouteStream(context.Background(), senderInbound("u1", "/status"), DeltaFunc(func(string) {})); err != nil { - t.Fatalf("RouteStream: %v", err) + if _, _, _, err := drainStreamed(t, streamChat(r), senderInbound("u1", "/status")); err != nil { + t.Fatalf("Stream: %v", err) } // Budget was 2; the local command must have consumed exactly one, so // exactly one more free-text message should still get through. - reply, err := r.RouteStream(context.Background(), senderInbound("u1", "hello"), DeltaFunc(func(string) {})) + _, kind, text, err := drainStreamed(t, streamChat(r), senderInbound("u1", "hello")) if err != nil { - t.Fatalf("RouteStream: %v", err) + t.Fatalf("Stream: %v", err) } - if reply != "streamed" { - t.Fatalf("second call reply = %q, want it to have budget left", reply) + if kind != "done" || text != "streamed" { + t.Fatalf("second call terminal = (%s, %q), want it to have budget left", kind, text) } - reply, err = r.RouteStream(context.Background(), senderInbound("u1", "hello again"), DeltaFunc(func(string) {})) + _, kind, text, err = drainStreamed(t, streamChat(r), senderInbound("u1", "hello again")) if err != nil { - t.Fatalf("RouteStream: %v", err) + t.Fatalf("Stream: %v", err) } - if reply != rateLimitReply { - t.Fatalf("third call reply = %q, want the budget exhausted after 2 charged calls", reply) + if kind != "done" || text != rateLimitReply { + t.Fatalf("third call terminal = (%s, %q), want the budget exhausted after 2 charged calls", kind, text) } } -func TestRouteStreamRateLimitBlocksStreamedReply(t *testing.T) { +func TestStreamRateLimitBlocksStreamedReply(t *testing.T) { r := NewRouter(nil, nil, "test") r.Limiter = ratelimit.New(time.Minute, 1) // The batcher is off: the blocked call must not wait on a batch window. @@ -187,15 +186,15 @@ func TestRouteStreamRateLimitBlocksStreamedReply(t *testing.T) { return "streamed", nil } - if _, err := r.RouteStream(context.Background(), senderInbound("u1", "hello"), DeltaFunc(func(string) {})); err != nil { - t.Fatalf("RouteStream: %v", err) + if _, _, _, err := drainStreamed(t, streamChat(r), senderInbound("u1", "hello")); err != nil { + t.Fatalf("Stream: %v", err) } - reply, err := r.RouteStream(context.Background(), senderInbound("u1", "hello again"), DeltaFunc(func(string) {})) + _, kind, text, err := drainStreamed(t, streamChat(r), senderInbound("u1", "hello again")) if err != nil { - t.Fatalf("RouteStream: %v", err) + t.Fatalf("Stream: %v", err) } - if reply != rateLimitReply { - t.Fatalf("reply = %q, want the rate-limit reply", reply) + if kind != "done" || text != rateLimitReply { + t.Fatalf("terminal = (%s, %q), want the rate-limit reply", kind, text) } if streamCalls != 1 { t.Fatalf("LLMStream called %d times, want exactly 1 (the blocked call must not reach it)", streamCalls) diff --git a/internal/gateway/gateway_stream_test.go b/internal/gateway/gateway_stream_test.go index 90b014d42..7c717643a 100644 --- a/internal/gateway/gateway_stream_test.go +++ b/internal/gateway/gateway_stream_test.go @@ -6,29 +6,28 @@ import ( "testing" ) -// RouteStream must degrade to Route when no streaming responder is set, so -// adapters can always call it without first checking for support. -func TestRouteStreamFallsBackWhenNoStreamResponder(t *testing.T) { +// The streamed chat adapter must degrade to the blocking responder when no +// streaming responder is set, so the chat contract's Stream can always be +// called without first checking for support. The degrade lives in +// streamTurn's contract now that every streamed entry is the adapter's. +func TestStreamFallsBackWhenNoStreamResponder(t *testing.T) { r := NewRouter(nil, func(context.Context, Inbound) (string, error) { return "blocking reply", nil }, "telegram") - var deltas []string - got, err := r.RouteStream(context.Background(), inbound("", "hello"), DeltaFunc(func(d string) { - deltas = append(deltas, d) - })) + deltas, kind, text, err := drainStreamed(t, streamChat(r), inbound("", "hello")) if err != nil { - t.Fatalf("RouteStream: %v", err) + t.Fatalf("Stream: %v", err) } - if got != "blocking reply" { - t.Errorf("reply = %q, want %q", got, "blocking reply") + if kind != "done" || text != "blocking reply" { + t.Errorf("terminal event = (%s, %q), want (done, \"blocking reply\")", kind, text) } if len(deltas) != 0 { t.Errorf("expected no deltas without a stream responder, got %v", deltas) } } -func TestRouteStreamStreamsFreeText(t *testing.T) { +func TestStreamStreamsFreeText(t *testing.T) { r := NewRouter(nil, nil, "telegram") r.LLMStream = func(_ context.Context, _ Inbound, stream TurnStream) (string, error) { for _, d := range []string{"a", "b", "c"} { @@ -37,21 +36,18 @@ func TestRouteStreamStreamsFreeText(t *testing.T) { return "abc", nil } - var sb strings.Builder - got, err := r.RouteStream(context.Background(), inbound("", "hello"), DeltaFunc(func(d string) { - sb.WriteString(d) - })) + deltas, kind, text, err := drainStreamed(t, streamChat(r), inbound("", "hello")) if err != nil { - t.Fatalf("RouteStream: %v", err) + t.Fatalf("Stream: %v", err) } - if got != "abc" || sb.String() != "abc" { - t.Errorf("reply=%q deltas=%q, want both %q", got, sb.String(), "abc") + if kind != "done" || text != "abc" || strings.Join(deltas, "") != "abc" { + t.Errorf("terminal=(%s,%q) deltas=%q, want (done,\"abc\") rendering \"abc\"", kind, text, deltas) } } // Local commands answer from local state in one step, so they must not be // sent to the streaming responder. -func TestRouteStreamDoesNotStreamLocalCommands(t *testing.T) { +func TestStreamDoesNotStreamLocalCommands(t *testing.T) { r := NewRouter(nil, nil, "telegram") streamed := false r.LLMStream = func(context.Context, Inbound, TurnStream) (string, error) { @@ -59,15 +55,15 @@ func TestRouteStreamDoesNotStreamLocalCommands(t *testing.T) { return "", nil } - if _, err := r.RouteStream(context.Background(), inbound("", "/status"), DeltaFunc(func(string) {})); err != nil { - t.Fatalf("RouteStream: %v", err) + if _, _, _, err := drainStreamed(t, streamChat(r), inbound("", "/status")); err != nil { + t.Fatalf("Stream: %v", err) } if streamed { t.Error("/status must not be routed to the streaming responder") } } -func TestRouteStreamRejectsUnknownCommandLocally(t *testing.T) { +func TestStreamRejectsUnknownCommandLocally(t *testing.T) { r := NewRouter(nil, nil, "telegram") streamed := false r.LLMStream = func(context.Context, Inbound, TurnStream) (string, error) { @@ -75,14 +71,14 @@ func TestRouteStreamRejectsUnknownCommandLocally(t *testing.T) { return "ok", nil } - reply, err := r.RouteStream(context.Background(), inbound("", "/commands"), DeltaFunc(func(string) {})) + _, kind, text, err := drainStreamed(t, streamChat(r), inbound("", "/commands")) if err != nil { - t.Fatalf("RouteStream: %v", err) + t.Fatalf("Stream: %v", err) } if streamed { t.Error("unknown /commands must not reach the streaming responder") } - if !strings.Contains(reply, "/help") { - t.Fatalf("reply = %q, want /help guidance", reply) + if kind != "done" || !strings.Contains(text, "/help") { + t.Errorf("terminal = (%s, %q), want the local unknown-command reply with /help guidance", kind, text) } } diff --git a/internal/gateway/gateway_test.go b/internal/gateway/gateway_test.go index 683646b51..a75847afa 100644 --- a/internal/gateway/gateway_test.go +++ b/internal/gateway/gateway_test.go @@ -349,15 +349,18 @@ func TestRouterRejectsUnknownCommand(t *testing.T) { return "fabricated streaming behavior", nil } - reply, err := r.RouteStream(context.Background(), inbound("", text), DeltaFunc(func(string) {})) + _, kind, text, err := drainStreamed(t, streamChat(r), inbound("", text)) if err != nil { - t.Fatalf("RouteStream: %v", err) + t.Fatalf("Stream: %v", err) } if llmCalls != 0 { t.Fatalf("unknown command invoked LLM %d times", llmCalls) } - if !strings.Contains(reply, "/frobnicate") || !strings.Contains(reply, "/help") { - t.Errorf("reply = %q, want unknown command and /help guidance", reply) + if kind != "done" { + t.Fatalf("terminal kind = %s, want a local done reply", kind) + } + if !strings.Contains(text, "/frobnicate") || !strings.Contains(text, "/help") { + t.Errorf("reply = %q, want unknown command and /help guidance", text) } }) } diff --git a/internal/gateway/gateway_text_batch_test.go b/internal/gateway/gateway_text_batch_test.go index d42f30b8f..38f5d990b 100644 --- a/internal/gateway/gateway_text_batch_test.go +++ b/internal/gateway/gateway_text_batch_test.go @@ -29,13 +29,35 @@ func fragment(n int, sender, text string) Inbound { } } -// routeOutcome records one RouteStream caller's reply and how its own -// stream rendered, so the dispatcher can be told apart from fragments whose -// turn another call covered. -type routeOutcome struct { - reply string +// streamOutcome records one streamed caller's rendered events, so the +// dispatcher -- started, its deltas, its done -- can be told apart from a +// covered fragment whose stream renders nothing at all. +type streamOutcome struct { + events []ChatEvent err error - deltas int +} + +// drained reads a streamed turn's events until its channel closes. The +// deadline is only a hang watchdog: a turn whose dispatch never lands must +// fail the test with what it rendered so far, never stall the suite. The +// timeout is returned, not raised with FailNow: draining runs on a helper +// goroutine, and FailNow from one strands the outcome channel. +func drained(events <-chan ChatEvent) ([]ChatEvent, error) { + var got []ChatEvent + deadline := time.Now().Add(5 * time.Second) + for { + select { + case event, ok := <-events: + if !ok { + return got, nil + } + got = append(got, event) + case <-time.After(time.Second): + } + if time.Now().After(deadline) { + return got, fmt.Errorf("the streamed turn never closed its events; rendered so far %+v", got) + } + } } // batchOf reads the open batch for a key, if one is open. Test-only: it is @@ -165,35 +187,57 @@ func TestRapidFireFragmentsTakeOneTurn(t *testing.T) { return "the answer", nil } + chat := &LocalChatAdapter{Router: r} key := batchKey{Platform: "telegram", Conversation: "chat:1", SenderID: "sam"} texts := []string{"first think", "about it", "then act"} - outcomes := make(chan routeOutcome, len(texts)) + outcomes := make(chan streamOutcome, len(texts)) for i, text := range texts { go func(i int, text string) { - deltas := 0 - reply, err := r.RouteStream(context.Background(), fragment(i, "sam", text), DeltaFunc(func(string) { - deltas++ - })) - outcomes <- routeOutcome{reply: reply, err: err, deltas: deltas} + events, err := chat.Stream(context.Background(), fragment(i, "sam", text)) + if err != nil { + outcomes <- streamOutcome{err: err} + return + } + rendered, drainErr := drained(events) + outcomes <- streamOutcome{events: rendered, err: drainErr} }(i, text) waitForFragments(t, r.Batches, key, i+1) } clock.Advance(2 * time.Hour) - answerers := 0 + dispatchers, covered := 0, 0 for range texts { outcome := <-outcomes if outcome.err != nil { - t.Errorf("RouteStream: %v", outcome.err) + t.Errorf("Stream: %v", outcome.err) continue } - if outcome.reply == "the answer" { - answerers++ - if outcome.deltas == 0 { + reply, sawStarted, sawDelta := "", false, false + for _, event := range outcome.events { + switch event.Kind { + case "started": + sawStarted = true + case "delta": + sawDelta = true + case "done": + reply = event.Text + case "error": + reply = "error: " + event.Text + } + } + switch { + case reply == "the answer": + dispatchers++ + if !sawStarted { + t.Error("the turn that ran must open its stream with a started event") + } + if !sawDelta { t.Error("the fragment whose turn ran must see the reply stream through it, but no delta arrived") } - } else if outcome.reply != "" { - t.Errorf("a covered fragment answered %q, want an empty reply", outcome.reply) + case len(outcome.events) != 0: + t.Errorf("a covered fragment's stream rendered %+v, want nothing at all", outcome.events) + default: + covered++ } } mu.Lock() @@ -201,8 +245,11 @@ func TestRapidFireFragmentsTakeOneTurn(t *testing.T) { if turns != 1 { t.Errorf("rapid-fire fragments from one sender started %d agent turns, want 1 (dispatched %q)", turns, dispatched) } - if answerers != 1 { - t.Errorf("%d of %d fragments returned the turn's own reply, want exactly 1 dispatcher", answerers, len(texts)) + if dispatchers != 1 { + t.Errorf("%d of %d fragments rendered the turn's own reply, want exactly 1 dispatcher", dispatchers, len(texts)) + } + if covered != len(texts)-1 { + t.Errorf("%d fragments rendered nothing, want %d covered ones", covered, len(texts)-1) } if turns == 1 && dispatched[0] != strings.Join(texts, "\n") { t.Errorf("the turn dispatched %q, want the fragments joined in arrival order", dispatched[0]) diff --git a/internal/gateway/message_dedup_test.go b/internal/gateway/message_dedup_test.go index 02e0c3daa..79f3a107b 100644 --- a/internal/gateway/message_dedup_test.go +++ b/internal/gateway/message_dedup_test.go @@ -221,16 +221,6 @@ func (c *countingLLM) respond(ctx context.Context, in Inbound) (string, error) { return c.reply, nil } -// countingStream is a TurnStream stub satisfying the interface, so the -// streaming router path runs without a renderer. -type countingStream struct { - deltas atomic.Int64 -} - -func (s *countingStream) Delta(text string) { s.deltas.Add(1) } -func (s *countingStream) ToolCall(event messaging.ToolCallEvent) {} -func (s *countingStream) Media(event messaging.MediaEvent) {} - // TestRouterRejectsRedeliveredMessage wires the gate where its owner // lives: the Router declines a second delivery of the same platform // message without reaching the LLM again, honours the TTL with an @@ -254,14 +244,20 @@ func TestRouterRejectsRedeliveredMessage(t *testing.T) { t.Fatalf("LLM called %d times for one accepted delivery, want 1", calls) } - stream := &countingStream{} - if reply, err := r.RouteStream(context.Background(), in, stream); err != nil { + // The streamed repeat must render the gate's prose, never the turn: + // the declines happen before any stream exists. + chat := &LocalChatAdapter{Router: r} + deltas, kind, text, err := drainStreamed(t, chat, in) + if err != nil { t.Fatalf("streamed repeat delivery: unexpected error %v", err) - } else if reply == "pong" { + } else if text == "pong" { t.Fatal("the streamed repeat delivery was streamed again instead of being declined") } - if calls != 1 || stream.deltas.Load() != 0 { - t.Fatalf("streamed repeat reached the stream (calls %d, deltas %d), want none", calls, stream.deltas.Load()) + if calls != 1 || len(deltas) != 0 { + t.Fatalf("streamed repeat reached the stream (calls %d, deltas %d), want none", calls, len(deltas)) + } + if kind != "done" || text != dedupReply { + t.Fatalf("streamed repeat terminal = (%s, %q), want the dedup prose", kind, text) } }) diff --git a/internal/gateway/session_titles_test.go b/internal/gateway/session_titles_test.go index 6cec0e184..8c1a8d18c 100644 --- a/internal/gateway/session_titles_test.go +++ b/internal/gateway/session_titles_test.go @@ -345,7 +345,7 @@ func TestRouteAutoTitleInFlightGuard(t *testing.T) { waitForTitle(t, store, sessionID, "Deploy worker") } -func TestRouteStreamAutoTitlesUntitledSession(t *testing.T) { +func TestStreamAutoTitlesUntitledSession(t *testing.T) { r := NewRouter(nil, nil, "test-gw") store := newFakeSessionStore() r.InitSessions(store) @@ -361,8 +361,8 @@ func TestRouteStreamAutoTitlesUntitledSession(t *testing.T) { t.Fatalf("resolve: %v", err) } gen.titles[sessionID] = "Stream topic" - if _, err := r.RouteStream(ctx, inboundFrom("ch", "u", "stream me"), DeltaFunc(func(string) {})); err != nil { - t.Fatalf("RouteStream: %v", err) + if _, _, _, err := drainStreamed(t, streamChat(r), inboundFrom("ch", "u", "stream me")); err != nil { + t.Fatalf("Stream: %v", err) } waitForTitle(t, store, sessionID, "Stream topic") } diff --git a/internal/gateway/stream_test.go b/internal/gateway/stream_test.go index 06004f135..9238232cf 100644 --- a/internal/gateway/stream_test.go +++ b/internal/gateway/stream_test.go @@ -25,7 +25,7 @@ func TestDeltaFuncIsTextOnly(t *testing.T) { } } -// RouteStream and TurnRunner accept a nil sink to mean "not streaming", so a +// streamTurn and TurnRunner accept a nil sink to mean "not streaming", so a // nil DeltaFunc must be inert rather than a panic waiting for the first token. func TestDeltaFuncNilIsInert(t *testing.T) { stream := DeltaFunc(nil)