From 037cc581f84621bcf479d50e1a96e340d4aa12bc Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Wed, 18 Nov 2026 00:36:35 +1100 Subject: [PATCH] fix(telegram): a long message goes out in parts instead of failing for ever The connector sent title and body as one sendMessage, so any notification over Telegram's 4096 UTF-16 unit limit was refused with 'message is too long' and, because a delivery is retried until the channel confirms it, retried every ten minutes for ever without ever arriving. The chat channel had its own splitter; both now share one in sdk/telegramtext, which counts UTF-16 units rather than runes so an emoji-heavy body isn't over the limit either. --- channels/telegram/main.go | 27 +-------- channels/telegram/main_test.go | 19 ------- connectors/telegram/telegram.go | 38 +++++++------ connectors/telegram/telegram_test.go | 84 +++++++++++++++++++++++++-- sdk/telegramtext/split.go | 85 ++++++++++++++++++++++++++++ sdk/telegramtext/split_test.go | 55 ++++++++++++++++++ 6 files changed, 243 insertions(+), 65 deletions(-) create mode 100644 sdk/telegramtext/split.go create mode 100644 sdk/telegramtext/split_test.go diff --git a/channels/telegram/main.go b/channels/telegram/main.go index a7554d5..08546d5 100644 --- a/channels/telegram/main.go +++ b/channels/telegram/main.go @@ -22,6 +22,7 @@ import ( "github.com/samcharles93/archipelago/sdk/channel" gatewayv1 "github.com/samcharles93/archipelago/sdk/gateway/v1" + "github.com/samcharles93/archipelago/sdk/telegramtext" ) const platform = "telegram" @@ -435,7 +436,7 @@ func helpText() string { // sendText sends one message, split under Telegram's 4096-character limit. func (g *gateway) sendText(ctx context.Context, chatID int64, threadID int, text string) { - for _, part := range splitText(text, messageMaxLen) { + for _, part := range telegramtext.Split(text, messageMaxLen) { params := &bot.SendMessageParams{ChatID: chatID, Text: part} if threadID != 0 { params.MessageThreadID = threadID @@ -447,7 +448,7 @@ func (g *gateway) sendText(ctx context.Context, chatID int64, threadID int, text } func (g *gateway) editText(ctx context.Context, chatID int64, messageID int, text string) { - parts := splitText(text, messageMaxLen) + parts := telegramtext.Split(text, messageMaxLen) if _, err := g.bot.EditMessageText(ctx, &bot.EditMessageTextParams{ChatID: chatID, MessageID: messageID, Text: parts[0]}); err != nil { g.log.DebugContext(ctx, "edit message failed", "chat_id", chatID, "error", err) } @@ -456,28 +457,6 @@ func (g *gateway) editText(ctx context.Context, chatID int64, messageID int, tex // messageMaxLen keeps one message under Telegram's 4096-character limit. const messageMaxLen = 4000 -// splitText splits on paragraph boundaries, then hard-wraps long lines. -func splitText(text string, max int) []string { - if len([]rune(text)) <= max { - return []string{text} - } - var parts []string - for _, para := range strings.Split(text, "\n\n") { - for len([]rune(para)) > max { - cut := max - if i := strings.LastIndex(para[:min(cut, len(para))], "\n"); i > 0 { - cut = i - } - parts = append(parts, para[:cut]) - para = strings.TrimLeft(para[cut:], "\n") - } - if para != "" { - parts = append(parts, para) - } - } - return parts -} - func (g *gateway) panicRecovery() bot.Middleware { return func(next bot.HandlerFunc) bot.HandlerFunc { return func(ctx context.Context, b *bot.Bot, update *models.Update) { diff --git a/channels/telegram/main_test.go b/channels/telegram/main_test.go index b1959b5..de2102c 100644 --- a/channels/telegram/main_test.go +++ b/channels/telegram/main_test.go @@ -70,25 +70,6 @@ func TestParseSettings(t *testing.T) { } } -func TestSplitText(t *testing.T) { - long := strings.Repeat("a", 9000) - parts := splitText(long, messageMaxLen) - if len(parts) != 3 { - t.Fatalf("splitText gave %d parts, want 3", len(parts)) - } - for _, p := range parts { - if len([]rune(p)) > messageMaxLen { - t.Errorf("part is %d runes, over the limit", len([]rune(p))) - } - } - if got := splitText("short", messageMaxLen); len(got) != 1 || got[0] != "short" { - t.Errorf("splitText(short) = %v", got) - } - paras := splitText("one\n\n"+long, messageMaxLen) - if paras[0] != "one" { - t.Errorf("first part = %q, want %q", paras[0], "one") - } -} func TestConversationKey(t *testing.T) { for _, tc := range []struct { diff --git a/connectors/telegram/telegram.go b/connectors/telegram/telegram.go index d3f6567..9b5c7f9 100644 --- a/connectors/telegram/telegram.go +++ b/connectors/telegram/telegram.go @@ -17,6 +17,7 @@ import ( "github.com/samcharles93/archipelago/sdk/connector" connectorv1 "github.com/samcharles93/archipelago/sdk/connector/v1" + "github.com/samcharles93/archipelago/sdk/telegramtext" ) const ( @@ -84,8 +85,10 @@ type message struct { ApprovalRequestID string `json:"approval_request_id"` } -// Deliver writes the message to each allowed user. Telegram can't dedupe a -// send, so a delivery fails only when no chat got it: retrying a partial one +// Deliver writes the message to each allowed user. One message holds at most +// 4096 UTF-16 code units, so a longer body goes out as several messages in +// order, with an approval request's buttons on the last. Telegram can't dedupe +// a send, so a delivery fails only when no chat got it: retrying a partial one // would repeat it to the users who already have it. func (t *telegram) Deliver(ctx context.Context, delivery string, input jsontext.Value, _ string) (string, error) { if delivery != "message" { @@ -101,22 +104,25 @@ func (t *telegram) Deliver(ctx context.Context, delivery string, input jsontext. } var ids []string var failed []error + parts := telegramtext.Split(text, telegramtext.MaxLen) for _, chat := range t.users { - body := map[string]any{"chat_id": chat, "text": text, "link_preview_options": map[string]any{"is_disabled": true}} - if m.ApprovalRequestID != "" { - body["reply_markup"] = map[string]any{"inline_keyboard": [][]map[string]any{{ - {"text": "Approve", "callback_data": answerPrefix + "yes:" + m.ApprovalRequestID}, - {"text": "Deny", "callback_data": answerPrefix + "no:" + m.ApprovalRequestID}, - }}} + for i, part := range parts { + body := map[string]any{"chat_id": chat, "text": part, "link_preview_options": map[string]any{"is_disabled": true}} + if i == len(parts)-1 && m.ApprovalRequestID != "" { + body["reply_markup"] = map[string]any{"inline_keyboard": [][]map[string]any{{ + {"text": "Approve", "callback_data": answerPrefix + "yes:" + m.ApprovalRequestID}, + {"text": "Deny", "callback_data": answerPrefix + "no:" + m.ApprovalRequestID}, + }}} + } + var sent struct { + MessageID int64 `json:"message_id"` + } + if err := t.call(ctx, "sendMessage", body, &sent); err != nil { + failed = append(failed, fmt.Errorf("chat %d: %w", chat, err)) + break + } + ids = append(ids, strconv.FormatInt(chat, 10)+"/"+strconv.FormatInt(sent.MessageID, 10)) } - var sent struct { - MessageID int64 `json:"message_id"` - } - if err := t.call(ctx, "sendMessage", body, &sent); err != nil { - failed = append(failed, fmt.Errorf("chat %d: %w", chat, err)) - continue - } - ids = append(ids, strconv.FormatInt(chat, 10)+"/"+strconv.FormatInt(sent.MessageID, 10)) } if len(ids) == 0 { return "", errors.Join(failed...) diff --git a/connectors/telegram/telegram_test.go b/connectors/telegram/telegram_test.go index d2f1ca5..4245c94 100644 --- a/connectors/telegram/telegram_test.go +++ b/connectors/telegram/telegram_test.go @@ -5,26 +5,37 @@ import ( json "encoding/json/v2" "net/http" "net/http/httptest" + "strconv" "strings" "sync" "testing" "github.com/samcharles93/archipelago/sdk/connector" + "github.com/samcharles93/archipelago/sdk/telegramtext" ) const fakeToken = "123:plainly-fake" //nolint:gosec // a fake bot token the test checks never leaks -// fakeBot serves one batch of updates, then nothing, and records the replies -// to taps. +// fakeBot serves one batch of updates, then nothing, records the replies to +// taps and the messages delivered, and refuses a message over Telegram's limit +// the way the Bot API does. type fakeBot struct { - mu sync.Mutex - updates string - served bool - replies []string + mu sync.Mutex + updates string + served bool + replies []string + sent []sentMessage + lastError string // idle is called when the batch has been served and fetched past. idle func() } +// sentMessage is one sendMessage the fake bot accepted. +type sentMessage struct { + Text string + Button bool +} + func (f *fakeBot) ServeHTTP(w http.ResponseWriter, r *http.Request) { f.mu.Lock() defer f.mu.Unlock() @@ -37,6 +48,20 @@ func (f *fakeBot) ServeHTTP(w http.ResponseWriter, r *http.Request) { f.idle() } _, _ = w.Write([]byte(`{"ok":true,"result":` + result + `}`)) + case strings.HasSuffix(r.URL.Path, "/sendMessage"): + var body struct { + Text string `json:"text"` + ReplyMark map[string]any `json:"reply_markup"` + } + _ = json.UnmarshalRead(r.Body, &body) + if telegramtext.Units(body.Text) > telegramtext.MaxLen { + f.lastError = "sendMessage: Bad Request: message is too long" + _, _ = w.Write([]byte(`{"ok":false,"error_code":400,"description":"Bad Request: message is too long"}`)) + return + } + f.sent = append(f.sent, sentMessage{Text: body.Text, Button: body.ReplyMark != nil}) + id := len(f.sent) + _, _ = w.Write([]byte(`{"ok":true,"result":{"message_id":` + strconv.Itoa(id) + `}}`)) case strings.HasSuffix(r.URL.Path, "/answerCallbackQuery"): var body struct { Text string `json:"text"` @@ -107,6 +132,53 @@ func TestOnlyAnAllowedUsersTapIsAnAnswer(t *testing.T) { } } +// TestALongMessageGoesOutInParts guards the delivery of a long notification: the +// Bot API refuses one message over 4096 UTF-16 code units, and a delivery the +// host can never complete is retried for ever, so the body goes out as several +// messages in order, with the request's buttons on the last one. +func TestALongMessageGoesOutInParts(t *testing.T) { + body := strings.Repeat("the findings run to a length no single message holds. ", 300) + bot := &fakeBot{} + srv := httptest.NewServer(bot) + defer srv.Close() + tg, err := newTelegram(connector.Config{ + Settings: map[string]string{"user_ids": "42"}, Secrets: map[string]string{"bot.token": fakeToken}, + }) + if err != nil { + t.Fatal(err) + } + tg.api = srv.URL + input, err := json.Marshal(map[string]string{"title": "Investigation finished", "body": body, "approval_request_id": "req-1"}) + if err != nil { + t.Fatal(err) + } + if _, err := tg.Deliver(t.Context(), "message", input, "k"); err != nil { + t.Fatalf("deliver: %v (bot said %q)", err, bot.lastError) + } + if len(bot.sent) < 2 { + t.Fatalf("sent %d messages, want the body split over several", len(bot.sent)) + } + for i, m := range bot.sent { + if units := telegramtext.Units(m.Text); units > telegramtext.MaxLen { + t.Errorf("message %d is %d units, over the limit", i+1, units) + } + if last := i == len(bot.sent)-1; m.Button != last { + t.Errorf("message %d buttons = %v, want them on the last only", i+1, m.Button) + } + } + if got := strings.Join(textsOf(bot.sent), ""); got != "Investigation finished\n\n"+body { + t.Errorf("the parts carry %d characters, want the whole message in order", len(got)) + } +} + +func textsOf(sent []sentMessage) []string { + out := make([]string, 0, len(sent)) + for _, m := range sent { + out = append(out, m.Text) + } + return out +} + // TestAFailedCallNeverShowsTheToken guards the bot token: it travels in the // request URL, and a transport error must not carry that URL into a log. func TestAFailedCallNeverShowsTheToken(t *testing.T) { diff --git a/sdk/telegramtext/split.go b/sdk/telegramtext/split.go new file mode 100644 index 0000000..458196c --- /dev/null +++ b/sdk/telegramtext/split.go @@ -0,0 +1,85 @@ +// Package telegramtext holds the one rule both Telegram senders share: the Bot +// API accepts at most MaxLen UTF-16 code units in one message, so anything +// longer goes out as several messages, in order. +package telegramtext + +import ( + "unicode/utf8" +) + +// MaxLen is Telegram's limit on one message's text, in UTF-16 code units. A +// rune outside the basic plane, an emoji for instance, counts as two. +const MaxLen = 4096 + +// Split cuts text into parts of at most max UTF-16 code units each, so the +// parts joined back together are the text it was given. It breaks on a blank +// line first, then on a line break, so a part ends where the author meant to +// pause, and never inside a character. +func Split(text string, max int) []string { + if max <= 0 { + max = MaxLen + } + if Units(text) <= max { + return []string{text} + } + var parts []string + for Units(text) > max { + cut := cutAt(text, max) + parts = append(parts, text[:cut]) + text = text[cut:] + } + return append(parts, text) +} + +// cutAt returns the byte index to cut s at, in preference order: after the last +// blank line that fits in max units, after the last line break that fits, or +// after the last character that fits. +func cutAt(s string, max int) int { + var ( + used, after int + blank, line = -1, -1 + ) + for i, r := range s { + n := 1 + if r > 0xFFFF { + n = 2 + } + if used+n > max { + break + } + used += n + after = i + utf8.RuneLen(r) + if r != '\n' { + continue + } + if i > 0 && s[i-1] == '\n' { + blank = after + } else { + line = after + } + } + switch { + case blank > 0: + return blank + case line > 0: + return line + case after > 0: + return after + } + // max is too small to hold one character; cut it anyway rather than spin. + _, size := utf8.DecodeRuneInString(s) + return size +} + +// Units is the length Telegram measures: UTF-16 code units, not runes. +func Units(s string) int { + n := 0 + for _, r := range s { + if r > 0xFFFF { + n += 2 + continue + } + n++ + } + return n +} diff --git a/sdk/telegramtext/split_test.go b/sdk/telegramtext/split_test.go new file mode 100644 index 0000000..a1bad46 --- /dev/null +++ b/sdk/telegramtext/split_test.go @@ -0,0 +1,55 @@ +package telegramtext + +import ( + "strings" + "testing" + "unicode/utf8" +) + +// TestSplitKeepsEveryPartUnderTheLimit is the one behaviour both Telegram +// senders depend on: a long body goes out in parts, none of which the Bot API +// refuses, and together they still say everything. +func TestSplitKeepsEveryPartUnderTheLimit(t *testing.T) { + tests := []struct { + name string + text string + max int + parts int + // first is the part the splitter should have started with. + first string + }{ + {name: "a message that fits is one part", text: "short", max: MaxLen, parts: 1}, + {name: "a long paragraph is cut", text: strings.Repeat("a", 9000), max: 4000, parts: 3}, + {name: "a blank line ends a part", text: "one\n\n" + strings.Repeat("a", 5000), max: 4000, parts: 3, first: "one\n\n"}, + {name: "a line break is preferred to the middle of a word", text: strings.Repeat("hold a line\n", 500), max: 4000}, // An emoji is two UTF-16 units, so 3000 of them are 6000: cutting by + // runes alone would hand Telegram a message it rejects. + {name: "emoji count twice", text: strings.Repeat("😀", 3000), max: 4000, parts: 2}, + {name: "an emoji is never cut in half", text: strings.Repeat("😀", 4001)}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + max := tt.max + if max == 0 { + max = MaxLen + } + parts := Split(tt.text, max) + if tt.parts > 0 && len(parts) != tt.parts { + t.Fatalf("Split gave %d parts, want %d", len(parts), tt.parts) + } + for i, p := range parts { + if units := Units(p); units > max { + t.Errorf("part %d is %d units, over the %d limit", i+1, units, max) + } + if !utf8.ValidString(p) { + t.Errorf("part %d cuts a character in half", i+1) + } + } + if tt.first != "" && parts[0] != tt.first { + t.Errorf("first part = %q, want %q", parts[0], tt.first) + } + if joined := strings.Join(parts, ""); joined != tt.text { + t.Errorf("the parts carry %d characters, want the %d they were given", len(joined), len(tt.text)) + } + }) + } +}