refactor(gateway): delete the dead RouteStream entry point

This commit is contained in:
sam committed 2026-09-30 08:57:32 +10:00
1 parent abbc6678d0
commit 72d7badb5b
9 files changed
+186 -142

No files matched your search

+33
View File
@@ -5,6 +5,39 @@ import (
"testing"
)
// streamChat builds the local chat adapter the way the composition wires
// it, pointed at the router under test; it is the production streaming
// entry these streamed-turn tests drive.
func streamChat(r *Router) *LocalChatAdapter {
return &LocalChatAdapter{Router: r}
}
// drainStreamed runs one streamed turn through the production chat adapter
// and returns what its stream rendered: the deltas, then the terminal
// event's kind and text.
func drainStreamed(t *testing.T, chat *LocalChatAdapter, in Inbound) ([]string, string, string, error) {
t.Helper()
events, err := chat.Stream(context.Background(), in)
if err != nil {
return nil, "", "", err
}
rendered, drainErr := drained(events)
if drainErr != nil {
return nil, "", "", drainErr
}
var deltas []string
kind, text := "", ""
for _, event := range rendered {
switch event.Kind {
case "delta":
deltas = append(deltas, event.Text)
case "done", "error":
kind, text = event.Kind, event.Text
}
}
return deltas, kind, text, nil
}
func TestLocalChatSnapshotAndStream(t *testing.T) {
ctx := t.Context()
sessions := NewSessionStoreMemory()
+19 -49
View File
@@ -167,9 +167,9 @@ type Router struct {
// reporting a value it does not have.
Health HealthSource
LLM LLMResponder // nil = LLM not wired yet
// LLMStream is the optional streaming responder. When set, adapters
// that can render partial output (see RouteStream) show the reply as
// it generates; when nil, everything falls back to LLM.
// LLMStream is the optional streaming responder. When set, the streamed
// chat adapter (internal/gateway/chat_local.go) shows the reply as it
// generates; when nil, everything falls back to LLM.
LLMStream LLMStreamResponder
// Identity is the archie identity this router belongs to (empty in
// single-identity deployments). Propagated into SpawnRequest and
@@ -345,8 +345,9 @@ func (r *Router) CollectTurn(ctx context.Context, in Inbound) (Inbound, bool, er
}
// route is Route's continuation once the caller has already cleared the
// rate-limit check (or is deliberately skipping it, as RouteStream's
// local-command fallthrough does to avoid charging one message twice).
// rate-limit check (or is deliberately skipping it, as streamTurn's
// local-command fallthrough and the streamed adapter's degraded-to-non-
// streaming path do, to avoid charging one message twice).
func (r *Router) route(ctx context.Context, in Inbound) (string, error) {
text := strings.TrimSpace(in.Message.Text)
cmd, _ := parseCmd(text, r.gatewayName)
@@ -502,56 +503,25 @@ func (r *Router) dispatchSessionCommand(ctx context.Context, msg messaging.Messa
return "", false, nil
}
// RouteStream is Route for adapters that can render a reply progressively.
// stream receives each new fragment of an LLM reply as it is generated, plus
// each completed tool call, in the order the model produced them; the
// complete reply is returned as usual.
//
// Only free-text messages stream. Gateway-local commands (/status, /model,
// …) answer from local state in a single step, so they return through the
// normal path with no delta callbacks. When no streaming responder is
// configured, RouteStream is exactly Route -- callers get the whole reply
// at the end and simply never see a delta, so an adapter can always call
// RouteStream without checking first.
func (r *Router) RouteStream(ctx context.Context, in Inbound, stream TurnStream) (string, error) {
if r.LLMStream == nil || stream == nil {
return r.Route(ctx, in)
}
if r.duplicateDelivery(in) {
return dedupReply, nil
}
if r.checkRateLimit(in) {
return rateLimitReply, nil
}
return r.streamTurn(ctx, in, stream)
}
// streamTurn dispatches one streaming turn for an inbound whose per-message
// gates have already run. RouteStream splits it off so the text batcher's
// dispatcher can carry the joined payload for its whole batch through here
// without the gates answering a second time for a message they already saw
// -- in particular so the combined payload's first-fragment source ID, which
// the dedup gate already recorded, does not read back as a redelivery.
// streamTurn dispatches one streamed turn for an inbound the streamed chat
// adapter has already prepared: the per-message gates (dedup, rate limit)
// ran per fragment in the adapter's prelude, and the text batcher has
// settled the turn's payload -- prelude's dispatcher carries the joined
// payload for its whole batch here, so the gates answer no second time for
// messages they already saw, and in particular the combined payload's
// first-fragment source ID, which the dedup gate already recorded, does
// not read back as a redelivery.
func (r *Router) streamTurn(ctx context.Context, in Inbound, stream TurnStream) (string, error) {
if r.LLMStream == nil || stream == nil {
return r.route(ctx, in)
}
cmd, _ := parseCmd(strings.TrimSpace(in.Message.Text), r.gatewayName)
if isLocalCommand(cmd) || strings.HasPrefix(cmd, "/") {
return r.route(ctx, in)
}
ready, mine, err := r.CollectTurn(ctx, in)
if err != nil {
return "", err
}
if !mine {
// A covered fragment: the batch's one turn streams on the call that
// dispatched it, so this caller's stream renders nothing. The empty
// final reply is what every renderer drops (Telegram's finalizer has
// no message and abandons; a dashboard client shows the turn that
// actually ran).
return "", nil
}
reply, err := r.LLMStream(ctx, ready, stream)
reply, err := r.LLMStream(ctx, in, stream)
if err == nil {
r.maybeAutoTitle(ctx, ready.Message)
r.maybeAutoTitle(ctx, in.Message)
}
return reply, err
}
+22 -23
View File
@@ -140,11 +140,11 @@ func TestRouteBudgetKeyLimitsWithoutSenderID(t *testing.T) {
}
}
// TestRouteStreamLocalCommandChargesExactlyOnce pins RouteStream's local
// command fallthrough (into route, not Route) against double-charging the
// budget: a status query must consume the same one unit whether it goes
// through Route or RouteStream.
func TestRouteStreamLocalCommandChargesExactlyOnce(t *testing.T) {
// TestStreamLocalCommandChargesExactlyOnce pins the streamed adapter's local
// command fallthrough (into route, not the streaming responder) against
// double-charging the budget: a status query must consume the same one unit
// whether it goes through Route or the adapter's Stream.
func TestStreamLocalCommandChargesExactlyOnce(t *testing.T) {
r := NewRouter(&fakeStore{counts: map[string]int{}}, nil, "test")
r.Limiter = ratelimit.New(time.Minute, 2)
// The batcher is off: the charge being pinned is per message, not per batch.
@@ -152,31 +152,30 @@ func TestRouteStreamLocalCommandChargesExactlyOnce(t *testing.T) {
r.LLMStream = func(context.Context, Inbound, TurnStream) (string, error) {
return "streamed", nil
}
if _, err := r.RouteStream(context.Background(), senderInbound("u1", "/status"), DeltaFunc(func(string) {})); err != nil {
t.Fatalf("RouteStream: %v", err)
if _, _, _, err := drainStreamed(t, streamChat(r), senderInbound("u1", "/status")); err != nil {
t.Fatalf("Stream: %v", err)
}
// Budget was 2; the local command must have consumed exactly one, so
// exactly one more free-text message should still get through.
reply, err := r.RouteStream(context.Background(), senderInbound("u1", "hello"), DeltaFunc(func(string) {}))
_, kind, text, err := drainStreamed(t, streamChat(r), senderInbound("u1", "hello"))
if err != nil {
t.Fatalf("RouteStream: %v", err)
t.Fatalf("Stream: %v", err)
}
if reply != "streamed" {
t.Fatalf("second call reply = %q, want it to have budget left", reply)
if kind != "done" || text != "streamed" {
t.Fatalf("second call terminal = (%s, %q), want it to have budget left", kind, text)
}
reply, err = r.RouteStream(context.Background(), senderInbound("u1", "hello again"), DeltaFunc(func(string) {}))
_, kind, text, err = drainStreamed(t, streamChat(r), senderInbound("u1", "hello again"))
if err != nil {
t.Fatalf("RouteStream: %v", err)
t.Fatalf("Stream: %v", err)
}
if reply != rateLimitReply {
t.Fatalf("third call reply = %q, want the budget exhausted after 2 charged calls", reply)
if kind != "done" || text != rateLimitReply {
t.Fatalf("third call terminal = (%s, %q), want the budget exhausted after 2 charged calls", kind, text)
}
}
func TestRouteStreamRateLimitBlocksStreamedReply(t *testing.T) {
func TestStreamRateLimitBlocksStreamedReply(t *testing.T) {
r := NewRouter(nil, nil, "test")
r.Limiter = ratelimit.New(time.Minute, 1)
// The batcher is off: the blocked call must not wait on a batch window.
@@ -187,15 +186,15 @@ func TestRouteStreamRateLimitBlocksStreamedReply(t *testing.T) {
return "streamed", nil
}
if _, err := r.RouteStream(context.Background(), senderInbound("u1", "hello"), DeltaFunc(func(string) {})); err != nil {
t.Fatalf("RouteStream: %v", err)
if _, _, _, err := drainStreamed(t, streamChat(r), senderInbound("u1", "hello")); err != nil {
t.Fatalf("Stream: %v", err)
}
reply, err := r.RouteStream(context.Background(), senderInbound("u1", "hello again"), DeltaFunc(func(string) {}))
_, kind, text, err := drainStreamed(t, streamChat(r), senderInbound("u1", "hello again"))
if err != nil {
t.Fatalf("RouteStream: %v", err)
t.Fatalf("Stream: %v", err)
}
if reply != rateLimitReply {
t.Fatalf("reply = %q, want the rate-limit reply", reply)
if kind != "done" || text != rateLimitReply {
t.Fatalf("terminal = (%s, %q), want the rate-limit reply", kind, text)
}
if streamCalls != 1 {
t.Fatalf("LLMStream called %d times, want exactly 1 (the blocked call must not reach it)", streamCalls)
+22 -26
View File
@@ -6,29 +6,28 @@ import (
"testing"
)
// RouteStream must degrade to Route when no streaming responder is set, so
// adapters can always call it without first checking for support.
func TestRouteStreamFallsBackWhenNoStreamResponder(t *testing.T) {
// The streamed chat adapter must degrade to the blocking responder when no
// streaming responder is set, so the chat contract's Stream can always be
// called without first checking for support. The degrade lives in
// streamTurn's contract now that every streamed entry is the adapter's.
func TestStreamFallsBackWhenNoStreamResponder(t *testing.T) {
r := NewRouter(nil, func(context.Context, Inbound) (string, error) {
return "blocking reply", nil
}, "telegram")
var deltas []string
got, err := r.RouteStream(context.Background(), inbound("", "hello"), DeltaFunc(func(d string) {
deltas = append(deltas, d)
}))
deltas, kind, text, err := drainStreamed(t, streamChat(r), inbound("", "hello"))
if err != nil {
t.Fatalf("RouteStream: %v", err)
t.Fatalf("Stream: %v", err)
}
if got != "blocking reply" {
t.Errorf("reply = %q, want %q", got, "blocking reply")
if kind != "done" || text != "blocking reply" {
t.Errorf("terminal event = (%s, %q), want (done, \"blocking reply\")", kind, text)
}
if len(deltas) != 0 {
t.Errorf("expected no deltas without a stream responder, got %v", deltas)
}
}
func TestRouteStreamStreamsFreeText(t *testing.T) {
func TestStreamStreamsFreeText(t *testing.T) {
r := NewRouter(nil, nil, "telegram")
r.LLMStream = func(_ context.Context, _ Inbound, stream TurnStream) (string, error) {
for _, d := range []string{"a", "b", "c"} {
@@ -37,21 +36,18 @@ func TestRouteStreamStreamsFreeText(t *testing.T) {
return "abc", nil
}
var sb strings.Builder
got, err := r.RouteStream(context.Background(), inbound("", "hello"), DeltaFunc(func(d string) {
sb.WriteString(d)
}))
deltas, kind, text, err := drainStreamed(t, streamChat(r), inbound("", "hello"))
if err != nil {
t.Fatalf("RouteStream: %v", err)
t.Fatalf("Stream: %v", err)
}
if got != "abc" || sb.String() != "abc" {
t.Errorf("reply=%q deltas=%q, want both %q", got, sb.String(), "abc")
if kind != "done" || text != "abc" || strings.Join(deltas, "") != "abc" {
t.Errorf("terminal=(%s,%q) deltas=%q, want (done,\"abc\") rendering \"abc\"", kind, text, deltas)
}
}
// Local commands answer from local state in one step, so they must not be
// sent to the streaming responder.
func TestRouteStreamDoesNotStreamLocalCommands(t *testing.T) {
func TestStreamDoesNotStreamLocalCommands(t *testing.T) {
r := NewRouter(nil, nil, "telegram")
streamed := false
r.LLMStream = func(context.Context, Inbound, TurnStream) (string, error) {
@@ -59,15 +55,15 @@ func TestRouteStreamDoesNotStreamLocalCommands(t *testing.T) {
return "", nil
}
if _, err := r.RouteStream(context.Background(), inbound("", "/status"), DeltaFunc(func(string) {})); err != nil {
t.Fatalf("RouteStream: %v", err)
if _, _, _, err := drainStreamed(t, streamChat(r), inbound("", "/status")); err != nil {
t.Fatalf("Stream: %v", err)
}
if streamed {
t.Error("/status must not be routed to the streaming responder")
}
}
func TestRouteStreamRejectsUnknownCommandLocally(t *testing.T) {
func TestStreamRejectsUnknownCommandLocally(t *testing.T) {
r := NewRouter(nil, nil, "telegram")
streamed := false
r.LLMStream = func(context.Context, Inbound, TurnStream) (string, error) {
@@ -75,14 +71,14 @@ func TestRouteStreamRejectsUnknownCommandLocally(t *testing.T) {
return "ok", nil
}
reply, err := r.RouteStream(context.Background(), inbound("", "/commands"), DeltaFunc(func(string) {}))
_, kind, text, err := drainStreamed(t, streamChat(r), inbound("", "/commands"))
if err != nil {
t.Fatalf("RouteStream: %v", err)
t.Fatalf("Stream: %v", err)
}
if streamed {
t.Error("unknown /commands must not reach the streaming responder")
}
if !strings.Contains(reply, "/help") {
t.Fatalf("reply = %q, want /help guidance", reply)
if kind != "done" || !strings.Contains(text, "/help") {
t.Errorf("terminal = (%s, %q), want the local unknown-command reply with /help guidance", kind, text)
}
}
+7 -4
View File
@@ -349,15 +349,18 @@ func TestRouterRejectsUnknownCommand(t *testing.T) {
return "fabricated streaming behavior", nil
}
reply, err := r.RouteStream(context.Background(), inbound("", text), DeltaFunc(func(string) {}))
_, kind, text, err := drainStreamed(t, streamChat(r), inbound("", text))
if err != nil {
t.Fatalf("RouteStream: %v", err)
t.Fatalf("Stream: %v", err)
}
if llmCalls != 0 {
t.Fatalf("unknown command invoked LLM %d times", llmCalls)
}
if !strings.Contains(reply, "/frobnicate") || !strings.Contains(reply, "/help") {
t.Errorf("reply = %q, want unknown command and /help guidance", reply)
if kind != "done" {
t.Fatalf("terminal kind = %s, want a local done reply", kind)
}
if !strings.Contains(text, "/frobnicate") || !strings.Contains(text, "/help") {
t.Errorf("reply = %q, want unknown command and /help guidance", text)
}
})
}
+68 -21
View File
@@ -29,13 +29,35 @@ func fragment(n int, sender, text string) Inbound {
}
}
// routeOutcome records one RouteStream caller's reply and how its own
// stream rendered, so the dispatcher can be told apart from fragments whose
// turn another call covered.
type routeOutcome struct {
reply string
// streamOutcome records one streamed caller's rendered events, so the
// dispatcher -- started, its deltas, its done -- can be told apart from a
// covered fragment whose stream renders nothing at all.
type streamOutcome struct {
events []ChatEvent
err error
deltas int
}
// drained reads a streamed turn's events until its channel closes. The
// deadline is only a hang watchdog: a turn whose dispatch never lands must
// fail the test with what it rendered so far, never stall the suite. The
// timeout is returned, not raised with FailNow: draining runs on a helper
// goroutine, and FailNow from one strands the outcome channel.
func drained(events <-chan ChatEvent) ([]ChatEvent, error) {
var got []ChatEvent
deadline := time.Now().Add(5 * time.Second)
for {
select {
case event, ok := <-events:
if !ok {
return got, nil
}
got = append(got, event)
case <-time.After(time.Second):
}
if time.Now().After(deadline) {
return got, fmt.Errorf("the streamed turn never closed its events; rendered so far %+v", got)
}
}
}
// batchOf reads the open batch for a key, if one is open. Test-only: it is
@@ -165,35 +187,57 @@ func TestRapidFireFragmentsTakeOneTurn(t *testing.T) {
return "the answer", nil
}
chat := &LocalChatAdapter{Router: r}
key := batchKey{Platform: "telegram", Conversation: "chat:1", SenderID: "sam"}
texts := []string{"first think", "about it", "then act"}
outcomes := make(chan routeOutcome, len(texts))
outcomes := make(chan streamOutcome, len(texts))
for i, text := range texts {
go func(i int, text string) {
deltas := 0
reply, err := r.RouteStream(context.Background(), fragment(i, "sam", text), DeltaFunc(func(string) {
deltas++
}))
outcomes <- routeOutcome{reply: reply, err: err, deltas: deltas}
events, err := chat.Stream(context.Background(), fragment(i, "sam", text))
if err != nil {
outcomes <- streamOutcome{err: err}
return
}
rendered, drainErr := drained(events)
outcomes <- streamOutcome{events: rendered, err: drainErr}
}(i, text)
waitForFragments(t, r.Batches, key, i+1)
}
clock.Advance(2 * time.Hour)
answerers := 0
dispatchers, covered := 0, 0
for range texts {
outcome := <-outcomes
if outcome.err != nil {
t.Errorf("RouteStream: %v", outcome.err)
t.Errorf("Stream: %v", outcome.err)
continue
}
if outcome.reply == "the answer" {
answerers++
if outcome.deltas == 0 {
reply, sawStarted, sawDelta := "", false, false
for _, event := range outcome.events {
switch event.Kind {
case "started":
sawStarted = true
case "delta":
sawDelta = true
case "done":
reply = event.Text
case "error":
reply = "error: " + event.Text
}
}
switch {
case reply == "the answer":
dispatchers++
if !sawStarted {
t.Error("the turn that ran must open its stream with a started event")
}
if !sawDelta {
t.Error("the fragment whose turn ran must see the reply stream through it, but no delta arrived")
}
} else if outcome.reply != "" {
t.Errorf("a covered fragment answered %q, want an empty reply", outcome.reply)
case len(outcome.events) != 0:
t.Errorf("a covered fragment's stream rendered %+v, want nothing at all", outcome.events)
default:
covered++
}
}
mu.Lock()
@@ -201,8 +245,11 @@ func TestRapidFireFragmentsTakeOneTurn(t *testing.T) {
if turns != 1 {
t.Errorf("rapid-fire fragments from one sender started %d agent turns, want 1 (dispatched %q)", turns, dispatched)
}
if answerers != 1 {
t.Errorf("%d of %d fragments returned the turn's own reply, want exactly 1 dispatcher", answerers, len(texts))
if dispatchers != 1 {
t.Errorf("%d of %d fragments rendered the turn's own reply, want exactly 1 dispatcher", dispatchers, len(texts))
}
if covered != len(texts)-1 {
t.Errorf("%d fragments rendered nothing, want %d covered ones", covered, len(texts)-1)
}
if turns == 1 && dispatched[0] != strings.Join(texts, "\n") {
t.Errorf("the turn dispatched %q, want the fragments joined in arrival order", dispatched[0])
+11 -15
View File
@@ -221,16 +221,6 @@ func (c *countingLLM) respond(ctx context.Context, in Inbound) (string, error) {
return c.reply, nil
}
// countingStream is a TurnStream stub satisfying the interface, so the
// streaming router path runs without a renderer.
type countingStream struct {
deltas atomic.Int64
}
func (s *countingStream) Delta(text string) { s.deltas.Add(1) }
func (s *countingStream) ToolCall(event messaging.ToolCallEvent) {}
func (s *countingStream) Media(event messaging.MediaEvent) {}
// TestRouterRejectsRedeliveredMessage wires the gate where its owner
// lives: the Router declines a second delivery of the same platform
// message without reaching the LLM again, honours the TTL with an
@@ -254,14 +244,20 @@ func TestRouterRejectsRedeliveredMessage(t *testing.T) {
t.Fatalf("LLM called %d times for one accepted delivery, want 1", calls)
}
stream := &countingStream{}
if reply, err := r.RouteStream(context.Background(), in, stream); err != nil {
// The streamed repeat must render the gate's prose, never the turn:
// the declines happen before any stream exists.
chat := &LocalChatAdapter{Router: r}
deltas, kind, text, err := drainStreamed(t, chat, in)
if err != nil {
t.Fatalf("streamed repeat delivery: unexpected error %v", err)
} else if reply == "pong" {
} else if text == "pong" {
t.Fatal("the streamed repeat delivery was streamed again instead of being declined")
}
if calls != 1 || stream.deltas.Load() != 0 {
t.Fatalf("streamed repeat reached the stream (calls %d, deltas %d), want none", calls, stream.deltas.Load())
if calls != 1 || len(deltas) != 0 {
t.Fatalf("streamed repeat reached the stream (calls %d, deltas %d), want none", calls, len(deltas))
}
if kind != "done" || text != dedupReply {
t.Fatalf("streamed repeat terminal = (%s, %q), want the dedup prose", kind, text)
}
})
+3 -3
View File
@@ -345,7 +345,7 @@ func TestRouteAutoTitleInFlightGuard(t *testing.T) {
waitForTitle(t, store, sessionID, "Deploy worker")
}
func TestRouteStreamAutoTitlesUntitledSession(t *testing.T) {
func TestStreamAutoTitlesUntitledSession(t *testing.T) {
r := NewRouter(nil, nil, "test-gw")
store := newFakeSessionStore()
r.InitSessions(store)
@@ -361,8 +361,8 @@ func TestRouteStreamAutoTitlesUntitledSession(t *testing.T) {
t.Fatalf("resolve: %v", err)
}
gen.titles[sessionID] = "Stream topic"
if _, err := r.RouteStream(ctx, inboundFrom("ch", "u", "stream me"), DeltaFunc(func(string) {})); err != nil {
t.Fatalf("RouteStream: %v", err)
if _, _, _, err := drainStreamed(t, streamChat(r), inboundFrom("ch", "u", "stream me")); err != nil {
t.Fatalf("Stream: %v", err)
}
waitForTitle(t, store, sessionID, "Stream topic")
}
+1 -1
View File
@@ -25,7 +25,7 @@ func TestDeltaFuncIsTextOnly(t *testing.T) {
}
}
// RouteStream and TurnRunner accept a nil sink to mean "not streaming", so a
// streamTurn and TurnRunner accept a nil sink to mean "not streaming", so a
// nil DeltaFunc must be inert rather than a panic waiting for the first token.
func TestDeltaFuncNilIsInert(t *testing.T) {
stream := DeltaFunc(nil)