diff --git a/internal/channels/telegram/live.go b/internal/channels/telegram/live.go index 703bc8071..16141ac56 100644 --- a/internal/channels/telegram/live.go +++ b/internal/channels/telegram/live.go @@ -98,6 +98,19 @@ type liveReply struct { rendered string messageID int last time.Time + + // terminal guards finalize and abandon so exactly one of them actually + // runs its body. Without this, a turn finishing naturally (finalize) + // and a concurrent Gateway.Stop (abandon, via abandonRestarted) can + // both reach their own Telegram edit for the same message: both edits + // go through, and whichever HTTP response Telegram processes last + // decides what the user sees, nondeterministically -- including a + // completed real answer getting silently overwritten by a restart + // marker. The loser's call blocks until the winner's body returns + // (sync.Once's own guarantee), then does nothing, so cleanup that must + // happen exactly once (stopRendering, forgetLive) is never skipped or + // duplicated regardless of which side wins. + terminal sync.Once } // newLiveReply creates a renderer bound to one chat/thread. showToolCalls is @@ -116,9 +129,107 @@ func (g *Gateway) newLiveReply(ctx context.Context, b *bot.Bot, chatID int64, me renderDone: make(chan struct{}), } go live.runRenderer(renderCtx) + g.registerLive(ctx, live) return live } +// resetLiveRegistry clears the previous launch's stopped mark. Called once +// per launch, alongside rebuilding turns: a /restart's Stop marks the +// registry stopped so nothing straggling from the outgoing bot instance +// gets silently registered into it, but the incoming instance needs a live +// registry of its own. +func (g *Gateway) resetLiveRegistry() { + g.liveMu.Lock() + g.liveStopped = false + g.liveMu.Unlock() +} + +// registerLive adds l to the set Stop drains on shutdown or restart. If a +// drain already claimed the registry for this launch, l is abandoned +// immediately instead of being registered: it arrived in the narrow window +// between that claim and this call, and without this it would sit in no +// registry, with nothing left to ever drain it. +func (g *Gateway) registerLive(ctx context.Context, l *liveReply) { + g.liveMu.Lock() + if g.liveStopped { + g.liveMu.Unlock() + go l.abandonRestarted(context.WithoutCancel(ctx)) + return + } + if g.liveReplies == nil { + g.liveReplies = make(map[*liveReply]struct{}) + } + g.liveReplies[l] = struct{}{} + g.liveMu.Unlock() +} + +// forgetLive removes l from the registry. Called at the top of doFinalize +// and doAbandon -- reached through l.terminal, so by the time this runs the +// caller has already exclusively won the right to decide this reply's +// outcome (see the terminal field doc) -- so a reply that completed on its +// own is never picked up a second time by a later Stop, and a reply Stop +// has already claimed is never double-processed. +func (g *Gateway) forgetLive(l *liveReply) { + g.liveMu.Lock() + defer g.liveMu.Unlock() + delete(g.liveReplies, l) +} + +// abandonAllLiveTimeout bounds how long Stop waits for in-flight replies to +// be marked before giving up and returning anyway. A single stalled +// Telegram API call must not hang gateway (and therefore daemon) shutdown +// indefinitely; entries still in flight past this deadline keep running in +// the background and finish (or die with the process) on their own. +const abandonAllLiveTimeout = 5 * time.Second + +// abandonAllLive drops the cursor, marked as interrupted, on every turn +// still streaming when the gateway stops or restarts. It exists because +// nothing else guarantees this happens: cancelling turnCtx does not +// guarantee the goroutine that owns a liveReply ever runs again to notice +// and clean up after itself, and a full daemon process exit gives it no +// chance to run at all, so this must not wait on that goroutine and must not +// skip a reply just because its owning turn is still technically running. +// +// Marking liveStopped and draining the registry happen together, under the +// same lock, so registerLive can never observe "stopped" without also +// observing an empty-of-this-entry registry, or vice versa. +func (g *Gateway) abandonAllLive(ctx context.Context) { + g.liveMu.Lock() + g.liveStopped = true + live := make([]*liveReply, 0, len(g.liveReplies)) + for l := range g.liveReplies { + live = append(live, l) + } + clear(g.liveReplies) + g.liveMu.Unlock() + + if len(live) == 0 { + return + } + + timeout := g.liveDrainTimeout + if timeout <= 0 { + timeout = abandonAllLiveTimeout + } + + done := make(chan struct{}) + go func() { + defer close(done) + var wg sync.WaitGroup + for _, l := range live { + wg.Go(func() { l.abandonRestarted(ctx) }) + } + wg.Wait() + }() + + select { + case <-done: + case <-time.After(timeout): + g.log.Warn("gateway stop: gave up waiting for in-flight replies to be marked as restarted", + "pending", len(live), "timeout", timeout) + } +} + func (l *liveReply) runRenderer(ctx context.Context) { defer close(l.renderDone) for { @@ -249,8 +360,18 @@ func (l *liveReply) markRendered(body string) { // still attached, forever. Doing this inside finalize rather than trusting // each caller to pass a live context is what keeps this fix from silently // regressing the next time a caller is added. +// +// The actual work runs inside l.terminal so a concurrent abandon (a gateway +// shutdown abandoning every in-flight reply while this turn is finishing +// naturally, see Gateway.abandonAllLive) cannot send its own conflicting +// edit for the same message -- see the terminal field doc. func (l *liveReply) finalize(ctx context.Context, reply string) { ctx = context.WithoutCancel(ctx) + l.terminal.Do(func() { l.doFinalize(ctx, reply) }) +} + +func (l *liveReply) doFinalize(ctx context.Context, reply string) { + l.g.forgetLive(l) // Finish any queued live frame before replacing it, then retire the // worker so no stale edit can race the authoritative delivery. @@ -301,10 +422,18 @@ const abandonMarker = "\n\n❌ _turn failed before finishing_" // Like finalize, it strips ctx of cancellation before sending: abandon runs // precisely when a /stop has cancelled turnCtx, so honouring that // cancellation in the very edit meant to drop the cursor would leave the -// cursor blinking on a message nothing will ever finish. +// cursor blinking on a message nothing will ever finish. And like finalize, +// the actual work runs inside l.terminal -- see the terminal field doc and +// finalize's. func (l *liveReply) abandon(ctx context.Context) { + ctx = context.WithoutCancel(ctx) + l.terminal.Do(func() { l.doAbandon(ctx) }) +} + +func (l *liveReply) doAbandon(ctx context.Context) { + l.g.forgetLive(l) l.stopRendering() - l.abandonStopped(context.WithoutCancel(ctx)) + l.abandonStopped(ctx) } // abandonFailed leaves a failed turn readable and visibly marked as failed, @@ -318,6 +447,25 @@ func (l *liveReply) abandonFailed(ctx context.Context) { l.abandon(ctx) } +// restartMarker distinguishes a gateway-restart abandon from both a clean +// /stop (abandon) and a provider failure (abandonFailed): the turn was cut +// off by an operational event rather than a fault in it or a deliberate +// stop, and unlike either of those the user's message is still worth +// resending once the gateway is back. +const restartMarker = "\n\n⚠️ _archie restarted before finishing -- send your message again_" + +// abandonRestarted leaves a turn interrupted by gateway shutdown or restart +// readable and marked, for the same reason abandonFailed marks a provider +// error: nothing will ever resume this turn, and an unmarked partial reads +// as a finished answer either way. Called by Gateway.abandonAllLive, never +// by the turn's own goroutine. +func (l *liveReply) abandonRestarted(ctx context.Context) { + l.mu.Lock() + l.answerBuf.WriteString(restartMarker) + l.mu.Unlock() + l.abandon(ctx) +} + func (l *liveReply) abandonStopped(ctx context.Context) { l.mu.Lock() answer := strings.TrimRight(l.answerBuf.String(), " \t\n") diff --git a/internal/channels/telegram/live_test.go b/internal/channels/telegram/live_test.go index 183bba96d..97d27367c 100644 --- a/internal/channels/telegram/live_test.go +++ b/internal/channels/telegram/live_test.go @@ -695,3 +695,192 @@ func TestLiveReplyAbandonStaysCleanUnlikeAbandonFailed(t *testing.T) { t.Fatalf("abandon (a /stop) was marked as a failure: %q", last.markdown) } } + +// A gateway restart or shutdown cancels turnCtx without any guarantee that +// the goroutine which owns a liveReply ever runs again to notice and clean +// up after itself -- a full process exit gives it no chance to run at all. +// abandonAllLive is what actually guarantees no message is stranded ending +// in the cursor: it must find every reply still mid-turn and mark it, +// without waiting on the turn's own goroutine. +func TestGatewayAbandonAllLiveMarksInFlightRepliesAsRestarted(t *testing.T) { + live, calls := newTestLiveReply(t, false) + + live.Delta("half an answer") + live.flushRendering() + + live.g.abandonAllLive(context.Background()) + + last := (*calls)[len(*calls)-1] + if !strings.Contains(last.markdown, "half an answer") { + t.Fatalf("abandonAllLive lost the streamed partial: %q", last.markdown) + } + if !strings.Contains(last.markdown, "restarted") { + t.Fatalf("abandonAllLive did not mark the message as interrupted by a restart: %q", last.markdown) + } + if n := len(live.g.liveReplies); n != 0 { + t.Fatalf("registry still holds %d entries after abandonAllLive, want 0", n) + } +} + +// A reply that already finished on its own must not be touched a second +// time by a Stop landing moments later: finalize/abandon deregister +// themselves, and abandonAllLive only acts on what is still registered. +func TestGatewayAbandonAllLiveSkipsAnAlreadyFinishedReply(t *testing.T) { + live, calls := newTestLiveReply(t, false) + + live.Delta("start") + live.finalize(context.Background(), "the finished answer") + callsBefore := len(*calls) + + live.g.abandonAllLive(context.Background()) + + if len(*calls) != callsBefore { + t.Fatalf("abandonAllLive sent %d more calls to an already-finished reply, want 0", len(*calls)-callsBefore) + } + last := (*calls)[len(*calls)-1] + if strings.Contains(last.markdown, "restarted") { + t.Fatalf("a finished reply was retroactively marked as restarted: %q", last.markdown) + } +} + +// Stop is the actual trigger for this cleanup in production: a running +// gateway must abandon every in-flight reply as part of stopping, not only +// when a test calls abandonAllLive directly. +func TestGatewayStopAbandonsInFlightLiveReplies(t *testing.T) { + live, calls := newTestLiveReply(t, false) + live.g.running = true + + live.Delta("still writing") + live.flushRendering() + + if err := live.g.Stop(context.Background()); err != nil { + t.Fatalf("Stop() = %v, want nil", err) + } + + last := (*calls)[len(*calls)-1] + if !strings.Contains(last.markdown, "restarted") { + t.Fatalf("Stop did not abandon the in-flight reply: %q", last.markdown) + } +} + +// A turn finishing naturally and a concurrent gateway shutdown can both +// reach the same liveReply at once: finalize from the turn's own goroutine, +// abandonRestarted from abandonAllLive. Without terminal serialising them, +// both send their own edit for the same message and whichever Telegram +// processes last wins nondeterministically -- including a real, complete +// answer getting silently overwritten by the restart marker. Exactly one of +// the two outcomes must win, never a mix and never neither. +func TestLiveReplyFinalizeAndAbandonRestartedAreMutuallyExclusive(t *testing.T) { + const iterations = 20 + for i := range iterations { + live, calls := newTestLiveReply(t, false) + live.Delta("streaming") + live.flushRendering() + + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + live.finalize(context.Background(), "THE REAL ANSWER") + }() + go func() { + defer wg.Done() + live.abandonRestarted(context.Background()) + }() + wg.Wait() + + last := (*calls)[len(*calls)-1] + hasAnswer := strings.Contains(last.markdown, "THE REAL ANSWER") + hasRestart := strings.Contains(last.markdown, "restarted") + if hasAnswer == hasRestart { + t.Fatalf("iteration %d: final message = %q, want exactly one of the real answer or the restart marker, never both or neither", + i, last.markdown) + } + } +} + +// abandonAllLive must not hang gateway Stop forever on one stalled Telegram +// API call: a shutdown that never completes because a single reply's edit +// never returns defeats the whole point of a bounded cleanup path. +func TestGatewayAbandonAllLiveDoesNotBlockOnAHungReply(t *testing.T) { + // release is closed explicitly at the end of the test body, before the + // api.Close cleanup runs -- closing it via t.Cleanup instead would race + // api.Close (which blocks until the in-flight, permanently-hung request + // completes) and deadlock the test itself. + release := make(chan struct{}) + api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-release + })) + t.Cleanup(api.Close) + + b, err := bot.New("1:test", bot.WithServerURL(api.URL), bot.WithSkipGetMe()) + if err != nil { + t.Fatalf("new test bot: %v", err) + } + g := New("1:test", "", "", []int64{42}, slog.New(slog.DiscardHandler)) + g.liveDrainTimeout = 50 * time.Millisecond + + live := g.newLiveReply(context.Background(), b, 7, 0, false) + t.Cleanup(live.stopRendering) + live.mu.Lock() + live.messageID = 55 // pretend a message already exists, forcing a hung edit call + live.answerBuf.WriteString("stuck mid-edit") + live.mu.Unlock() + + start := time.Now() + g.abandonAllLive(context.Background()) + elapsed := time.Since(start) + close(release) + + if elapsed > g.liveDrainTimeout+2*time.Second { + t.Fatalf("abandonAllLive took %v, want bounded near liveDrainTimeout (%v)", elapsed, g.liveDrainTimeout) + } +} + +// A liveReply created in the narrow window after abandonAllLive has already +// claimed the registry must still end up marked, not silently registered +// into (and then lost from) a registry nothing will ever drain again. +func TestGatewayRegisterLiveAfterStopAbandonsImmediately(t *testing.T) { + // registerLive's stopped path abandons on its own goroutine (see + // registerLive), so the assertion below must synchronize on that + // goroutine's completion rather than polling fakeAPI's shared calls + // slice, which is unsynchronized and only ever written from the single + // HTTP handler goroutine in every other test in this file. + type capturedRequest struct { + body string + } + received := make(chan capturedRequest, 1) + api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + call := capturedCall(t, r) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"ok":true,"result":{"message_id":55,"date":1,"chat":{"id":7,"type":"private"}}}`)) + received <- capturedRequest{body: call.markdown} + })) + t.Cleanup(api.Close) + b, err := bot.New("1:test", bot.WithServerURL(api.URL), bot.WithSkipGetMe()) + if err != nil { + t.Fatalf("new test bot: %v", err) + } + + g := New("1:test", "", "", []int64{42}, slog.New(slog.DiscardHandler)) + g.liveMu.Lock() + g.liveStopped = true + g.liveMu.Unlock() + + live := g.newLiveReply(context.Background(), b, 7, 0, false) + t.Cleanup(live.stopRendering) + live.interval = 0 + + if n := len(g.liveReplies); n != 0 { + t.Fatalf("registry holds %d entries after a post-stop registration, want 0", n) + } + + select { + case req := <-received: + if !strings.Contains(req.body, "restarted") { + t.Fatalf("post-stop registration produced %q, want the restart marker", req.body) + } + case <-time.After(time.Second): + t.Fatal("a liveReply registered after stop was never abandoned") + } +} diff --git a/internal/channels/telegram/telegram.go b/internal/channels/telegram/telegram.go index 405c0b7bf..e48d7a4eb 100644 --- a/internal/channels/telegram/telegram.go +++ b/internal/channels/telegram/telegram.go @@ -91,6 +91,28 @@ type Gateway struct { // so a restart abandons in-flight turns with the old bot instance. turns *gateway.Turns + // liveMu guards liveReplies and liveStopped. + liveMu sync.Mutex + // liveReplies tracks every liveReply currently mid-turn. Cancelling + // turnCtx (a /restart, a daemon shutdown) does not guarantee the + // goroutine that owns a liveReply ever runs again to notice and clean + // up after itself -- a full process exit gives it no chance at all -- + // so Stop drains this registry directly rather than waiting on that + // goroutine. See abandonAllLive. The map itself survives a /restart + // (only turns above is rebuilt per launch); liveStopped is what scopes + // draining to one launch's lifecycle. + liveReplies map[*liveReply]struct{} + // liveDrainTimeout bounds abandonAllLive's wait for in-flight replies to + // be marked. Zero uses abandonAllLiveTimeout; tests shrink it so a + // deliberately-hung reply doesn't make the suite slow. + liveDrainTimeout time.Duration + // liveStopped is set the instant abandonAllLive claims the registry, so + // a newLiveReply that arrives in the narrow window between that claim + // and its caller actually registering is abandoned immediately instead + // of being added to a registry nothing will ever drain again. Reset to + // false at the start of the next launch. + liveStopped bool + log *slog.Logger bot *bot.Bot webhookCancel context.CancelFunc @@ -217,6 +239,7 @@ func (g *Gateway) launch(ctx context.Context, router *gateway.Router) (*bot.Bot, // context, which is this launch's, so a /restart cancels everything // still running under the outgoing bot instance. g.turns = gateway.NewTurns(g.log) + g.resetLiveRegistry() opts := []bot.Option{ bot.WithErrorsHandler(g.pipelineErrorHandler(ctx)), @@ -331,6 +354,8 @@ func (g *Gateway) Stop(ctx context.Context) error { g.log.Info("stopping telegram gateway") g.running = false + g.abandonAllLive(ctx) + if g.webhookCancel != nil { g.webhookCancel() g.webhookCancel = nil