From 76d13b76ceaf5a67c381637a45b19352d4f40901 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Fri, 28 Aug 2026 00:22:00 +1000 Subject: [PATCH] fix(telegram): drop retry_after bandaid, bump go-telegram/bot to v1.24.0 go-telegram/bot v1.24.0 fixes getUpdates to honour Telegram's retry_after on a 429 instead of its own 100ms-doubling-to-5s backoff (release notes: https://github.com/go-telegram/bot/releases/tag/v1.24.0). pipelineErrorHandler was our workaround for that upstream gap -- it now just logs. --- go.mod | 2 +- go.sum | 4 +- internal/channels/telegram/ratelimit.go | 47 +++---------- internal/channels/telegram/ratelimit_test.go | 74 +++++--------------- 4 files changed, 30 insertions(+), 97 deletions(-) diff --git a/go.mod b/go.mod index 255ef6549..909b99aec 100644 --- a/go.mod +++ b/go.mod @@ -65,7 +65,7 @@ require ( require ( code.gitea.io/sdk/gitea v0.25.1 github.com/go-git/go-git/v6 v6.0.0-alpha.5 - github.com/go-telegram/bot v1.23.0 + github.com/go-telegram/bot v1.24.0 github.com/google/codesearch v1.3.0-rc.1 github.com/google/uuid v1.6.0 github.com/moby/moby/api v1.55.0 diff --git a/go.sum b/go.sum index 4b7704e63..37aed04b6 100644 --- a/go.sum +++ b/go.sum @@ -56,8 +56,8 @@ github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-telegram/bot v1.23.0 h1:CKKQq115G/GUGBG8uuWl5uXbiBHyVjZBp/qqOLWZjJk= -github.com/go-telegram/bot v1.23.0/go.mod h1:i2TRs7fXWIeaceF3z7KzsMt/he0TwkVC680mvdTFYeM= +github.com/go-telegram/bot v1.24.0 h1:W/tWeUm8pkVkCGdf0n00fzJD/D31iAJ9RHFvIAZ2G9M= +github.com/go-telegram/bot v1.24.0/go.mod h1:i2TRs7fXWIeaceF3z7KzsMt/he0TwkVC680mvdTFYeM= github.com/google/codesearch v1.3.0-rc.1 h1:vSFGuF5wS3sjimuLzqPG+PGjWQ9VvEtPFbxdZD7YuuQ= github.com/google/codesearch v1.3.0-rc.1/go.mod h1:hKdpx7wOsvk2WLT/D6NLPxFeLT8jKCehZu21JpA2rIY= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= diff --git a/internal/channels/telegram/ratelimit.go b/internal/channels/telegram/ratelimit.go index 35214b5df..4cb63aadd 100644 --- a/internal/channels/telegram/ratelimit.go +++ b/internal/channels/telegram/ratelimit.go @@ -3,49 +3,24 @@ package telegram import ( "context" "errors" - "time" "github.com/go-telegram/bot" ) -// maxRetryAfter bounds how long a 429 can park the update loop. Telegram's -// retry_after is normally single-digit seconds; the cap stops a hostile or -// buggy value from stalling the gateway indefinitely. -const maxRetryAfter = 60 * time.Second - -// pipelineErrorHandler logs update-loop failures and, on a rate limit, waits -// the interval Telegram asked for. +// pipelineErrorHandler logs update-loop failures. // -// go-telegram/bot backs off on its own schedule -- 100ms, doubling, capped at -// 5s (get_updates.go incErrTimeout) -- and never reads RetryAfter. Against a -// "retry_after: 5" that means roughly a dozen further requests inside the -// window Telegram told us to stay silent for, which is what escalates a brief -// 429 into sustained 429s, 502s and request timeouts. The library calls this -// handler synchronously from the polling loop before applying its own backoff, -// so blocking here is the supported way to honour the interval without -// patching the dependency. -func (g *Gateway) pipelineErrorHandler(ctx context.Context) func(error) { +// go-telegram/bot v1.24.0 fixed getUpdates to honour retry_after on a 429 +// itself (https://github.com/go-telegram/bot/releases/tag/v1.24.0), instead +// of retrying on its own 100ms-doubling-to-5s schedule regardless of what +// Telegram asked for. This handler used to block for RetryAfter itself to +// compensate; that workaround is gone now that the library does it. +func (g *Gateway) pipelineErrorHandler(_ context.Context) func(error) { return func(err error) { - // errors.As unwraps the library's fmt.Errorf("error get updates, %w"). - var tooMany *bot.TooManyRequestsError - if !errors.As(err, &tooMany) { - g.log.Error("telegram pipeline error", "error", err) + if tooMany, ok := errors.AsType[*bot.TooManyRequestsError](err); ok { + g.log.Warn("telegram rate limited", "retry_after", tooMany.RetryAfter, + "component", "gateway-telegram") return } - - wait := time.Duration(tooMany.RetryAfter) * time.Second - if wait <= 0 { - wait = time.Second - } - wait = min(wait, maxRetryAfter) - - g.log.Warn("telegram rate limited, pausing update loop", - "retry_after", wait.String(), - "component", "gateway-telegram") - - select { - case <-ctx.Done(): - case <-time.After(wait): - } + g.log.Error("telegram pipeline error", "error", err) } } diff --git a/internal/channels/telegram/ratelimit_test.go b/internal/channels/telegram/ratelimit_test.go index 1769a6c0a..777aeaded 100644 --- a/internal/channels/telegram/ratelimit_test.go +++ b/internal/channels/telegram/ratelimit_test.go @@ -11,41 +11,27 @@ import ( "github.com/go-telegram/bot" ) -// TestPipelineErrorHandlerHonoursRetryAfter pins the fix for the carina -// incident: go-telegram/bot backs off 100ms and doubles, ignoring the -// retry_after Telegram sends, so a 429 was answered with a burst of further -// requests inside the window we were told to stay quiet for. -func TestPipelineErrorHandlerHonoursRetryAfter(t *testing.T) { +// TestPipelineErrorHandlerDoesNotBlock pins that the handler is a pure logger +// post go-telegram/bot v1.24.0: the library now honours retry_after itself +// (https://github.com/go-telegram/bot/releases/tag/v1.24.0), so the handler +// that used to block for the stated interval on our side would double-wait. +func TestPipelineErrorHandlerDoesNotBlock(t *testing.T) { tests := []struct { - name string - err error - minWait time.Duration - maxWait time.Duration + name string + err error }{ { - name: "bare rate limit error waits the stated interval", - err: &bot.TooManyRequestsError{RetryAfter: 1}, - minWait: time.Second, - maxWait: 3 * time.Second, + name: "bare rate limit error", + err: &bot.TooManyRequestsError{RetryAfter: 30}, }, { - // The library wraps it: fmt.Errorf("error get updates, %w", err). - name: "wrapped rate limit error is still recognised", - err: fmt.Errorf("error get updates, %w", &bot.TooManyRequestsError{RetryAfter: 1}), - minWait: time.Second, - maxWait: 3 * time.Second, + // The library wraps it: fmt.Errorf("error get updates, %w"). + name: "wrapped rate limit error is still recognised", + err: fmt.Errorf("error get updates, %w", &bot.TooManyRequestsError{RetryAfter: 30}), }, { - name: "zero retry_after still pauses rather than spinning", - err: &bot.TooManyRequestsError{RetryAfter: 0}, - minWait: time.Second, - maxWait: 3 * time.Second, - }, - { - name: "unrelated errors are logged without pausing", - err: errors.New("502 Bad Gateway"), - minWait: 0, - maxWait: 250 * time.Millisecond, + name: "unrelated errors are logged without pausing", + err: errors.New("502 Bad Gateway"), }, } @@ -58,37 +44,9 @@ func TestPipelineErrorHandlerHonoursRetryAfter(t *testing.T) { handle(tc.err) elapsed := time.Since(start) - if elapsed < tc.minWait { - t.Errorf("returned after %s, want at least %s", elapsed, tc.minWait) - } - if elapsed > tc.maxWait { - t.Errorf("returned after %s, want at most %s", elapsed, tc.maxWait) + if elapsed > 250*time.Millisecond { + t.Errorf("returned after %s, want immediate return (no local backoff)", elapsed) } }) } } - -// TestPipelineErrorHandlerReleasesOnShutdown pins that a rate-limit pause does -// not hold the update loop open past cancellation -- otherwise a /restart or -// SIGTERM during a 429 would block for the full retry_after. -func TestPipelineErrorHandlerReleasesOnShutdown(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - g := &Gateway{log: slog.New(slog.DiscardHandler)} - handle := g.pipelineErrorHandler(ctx) - - cancel() - start := time.Now() - handle(&bot.TooManyRequestsError{RetryAfter: 30}) - - if elapsed := time.Since(start); elapsed > 250*time.Millisecond { - t.Errorf("blocked %s after cancellation, want immediate return", elapsed) - } -} - -// TestRetryAfterIsBounded pins the ceiling on how long a single 429 can park -// the loop, so an implausible retry_after cannot stall the gateway. -func TestRetryAfterIsBounded(t *testing.T) { - if maxRetryAfter > 5*time.Minute { - t.Errorf("maxRetryAfter = %s, too long to leave the gateway unresponsive", maxRetryAfter) - } -}