fix(telegram): a long message goes out in parts instead of failing for ever
publish / publish (push) Successful in 1m22s

The connector sent title and body as one sendMessage, so any notification
over Telegram's 4096 UTF-16 unit limit was refused with 'message is too
long' and, because a delivery is retried until the channel confirms it,
retried every ten minutes for ever without ever arriving. The chat channel
had its own splitter; both now share one in sdk/telegramtext, which counts
UTF-16 units rather than runes so an emoji-heavy body isn't over the limit
either.
This commit is contained in:
sam committed 2026-11-18 00:36:44 +11:00
1 parent 554fe5891f
commit 48019ec868
6 files changed
+243 -66

No files matched your search

+3 -24
View File
@@ -22,6 +22,7 @@ import (
"github.com/samcharles93/archipelago/sdk/channel" "github.com/samcharles93/archipelago/sdk/channel"
gatewayv1 "github.com/samcharles93/archipelago/sdk/gateway/v1" gatewayv1 "github.com/samcharles93/archipelago/sdk/gateway/v1"
"github.com/samcharles93/archipelago/sdk/telegramtext"
) )
const platform = "telegram" const platform = "telegram"
@@ -435,7 +436,7 @@ func helpText() string {
// sendText sends one message, split under Telegram's 4096-character limit. // sendText sends one message, split under Telegram's 4096-character limit.
func (g *gateway) sendText(ctx context.Context, chatID int64, threadID int, text string) { func (g *gateway) sendText(ctx context.Context, chatID int64, threadID int, text string) {
for _, part := range splitText(text, messageMaxLen) { for _, part := range telegramtext.Split(text, messageMaxLen) {
params := &bot.SendMessageParams{ChatID: chatID, Text: part} params := &bot.SendMessageParams{ChatID: chatID, Text: part}
if threadID != 0 { if threadID != 0 {
params.MessageThreadID = threadID params.MessageThreadID = threadID
@@ -447,7 +448,7 @@ func (g *gateway) sendText(ctx context.Context, chatID int64, threadID int, text
} }
func (g *gateway) editText(ctx context.Context, chatID int64, messageID int, text string) { func (g *gateway) editText(ctx context.Context, chatID int64, messageID int, text string) {
parts := splitText(text, messageMaxLen) parts := telegramtext.Split(text, messageMaxLen)
if _, err := g.bot.EditMessageText(ctx, &bot.EditMessageTextParams{ChatID: chatID, MessageID: messageID, Text: parts[0]}); err != nil { if _, err := g.bot.EditMessageText(ctx, &bot.EditMessageTextParams{ChatID: chatID, MessageID: messageID, Text: parts[0]}); err != nil {
g.log.DebugContext(ctx, "edit message failed", "chat_id", chatID, "error", err) g.log.DebugContext(ctx, "edit message failed", "chat_id", chatID, "error", err)
} }
@@ -456,28 +457,6 @@ func (g *gateway) editText(ctx context.Context, chatID int64, messageID int, tex
// messageMaxLen keeps one message under Telegram's 4096-character limit. // messageMaxLen keeps one message under Telegram's 4096-character limit.
const messageMaxLen = 4000 const messageMaxLen = 4000
// splitText splits on paragraph boundaries, then hard-wraps long lines.
func splitText(text string, max int) []string {
if len([]rune(text)) <= max {
return []string{text}
}
var parts []string
for _, para := range strings.Split(text, "\n\n") {
for len([]rune(para)) > max {
cut := max
if i := strings.LastIndex(para[:min(cut, len(para))], "\n"); i > 0 {
cut = i
}
parts = append(parts, para[:cut])
para = strings.TrimLeft(para[cut:], "\n")
}
if para != "" {
parts = append(parts, para)
}
}
return parts
}
func (g *gateway) panicRecovery() bot.Middleware { func (g *gateway) panicRecovery() bot.Middleware {
return func(next bot.HandlerFunc) bot.HandlerFunc { return func(next bot.HandlerFunc) bot.HandlerFunc {
return func(ctx context.Context, b *bot.Bot, update *models.Update) { return func(ctx context.Context, b *bot.Bot, update *models.Update) {
-20
View File
@@ -70,26 +70,6 @@ func TestParseSettings(t *testing.T) {
} }
} }
func TestSplitText(t *testing.T) {
long := strings.Repeat("a", 9000)
parts := splitText(long, messageMaxLen)
if len(parts) != 3 {
t.Fatalf("splitText gave %d parts, want 3", len(parts))
}
for _, p := range parts {
if len([]rune(p)) > messageMaxLen {
t.Errorf("part is %d runes, over the limit", len([]rune(p)))
}
}
if got := splitText("short", messageMaxLen); len(got) != 1 || got[0] != "short" {
t.Errorf("splitText(short) = %v", got)
}
paras := splitText("one\n\n"+long, messageMaxLen)
if paras[0] != "one" {
t.Errorf("first part = %q, want %q", paras[0], "one")
}
}
func TestConversationKey(t *testing.T) { func TestConversationKey(t *testing.T) {
for _, tc := range []struct { for _, tc := range []struct {
chatID, thread int64 chatID, thread int64
+22 -16
View File
@@ -17,6 +17,7 @@ import (
"github.com/samcharles93/archipelago/sdk/connector" "github.com/samcharles93/archipelago/sdk/connector"
connectorv1 "github.com/samcharles93/archipelago/sdk/connector/v1" connectorv1 "github.com/samcharles93/archipelago/sdk/connector/v1"
"github.com/samcharles93/archipelago/sdk/telegramtext"
) )
const ( const (
@@ -84,8 +85,10 @@ type message struct {
ApprovalRequestID string `json:"approval_request_id"` ApprovalRequestID string `json:"approval_request_id"`
} }
// Deliver writes the message to each allowed user. Telegram can't dedupe a // Deliver writes the message to each allowed user. One message holds at most
// send, so a delivery fails only when no chat got it: retrying a partial one // 4096 UTF-16 code units, so a longer body goes out as several messages in
// order, with an approval request's buttons on the last. Telegram can't dedupe
// a send, so a delivery fails only when no chat got it: retrying a partial one
// would repeat it to the users who already have it. // would repeat it to the users who already have it.
func (t *telegram) Deliver(ctx context.Context, delivery string, input jsontext.Value, _ string) (string, error) { func (t *telegram) Deliver(ctx context.Context, delivery string, input jsontext.Value, _ string) (string, error) {
if delivery != "message" { if delivery != "message" {
@@ -101,22 +104,25 @@ func (t *telegram) Deliver(ctx context.Context, delivery string, input jsontext.
} }
var ids []string var ids []string
var failed []error var failed []error
parts := telegramtext.Split(text, telegramtext.MaxLen)
for _, chat := range t.users { for _, chat := range t.users {
body := map[string]any{"chat_id": chat, "text": text, "link_preview_options": map[string]any{"is_disabled": true}} for i, part := range parts {
if m.ApprovalRequestID != "" { body := map[string]any{"chat_id": chat, "text": part, "link_preview_options": map[string]any{"is_disabled": true}}
body["reply_markup"] = map[string]any{"inline_keyboard": [][]map[string]any{{ if i == len(parts)-1 && m.ApprovalRequestID != "" {
{"text": "Approve", "callback_data": answerPrefix + "yes:" + m.ApprovalRequestID}, body["reply_markup"] = map[string]any{"inline_keyboard": [][]map[string]any{{
{"text": "Deny", "callback_data": answerPrefix + "no:" + m.ApprovalRequestID}, {"text": "Approve", "callback_data": answerPrefix + "yes:" + m.ApprovalRequestID},
}}} {"text": "Deny", "callback_data": answerPrefix + "no:" + m.ApprovalRequestID},
}}}
}
var sent struct {
MessageID int64 `json:"message_id"`
}
if err := t.call(ctx, "sendMessage", body, &sent); err != nil {
failed = append(failed, fmt.Errorf("chat %d: %w", chat, err))
break
}
ids = append(ids, strconv.FormatInt(chat, 10)+"/"+strconv.FormatInt(sent.MessageID, 10))
} }
var sent struct {
MessageID int64 `json:"message_id"`
}
if err := t.call(ctx, "sendMessage", body, &sent); err != nil {
failed = append(failed, fmt.Errorf("chat %d: %w", chat, err))
continue
}
ids = append(ids, strconv.FormatInt(chat, 10)+"/"+strconv.FormatInt(sent.MessageID, 10))
} }
if len(ids) == 0 { if len(ids) == 0 {
return "", errors.Join(failed...) return "", errors.Join(failed...)
+78 -6
View File
@@ -5,26 +5,37 @@ import (
json "encoding/json/v2" json "encoding/json/v2"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"strconv"
"strings" "strings"
"sync" "sync"
"testing" "testing"
"github.com/samcharles93/archipelago/sdk/connector" "github.com/samcharles93/archipelago/sdk/connector"
"github.com/samcharles93/archipelago/sdk/telegramtext"
) )
const fakeToken = "123:plainly-fake" //nolint:gosec // a fake bot token the test checks never leaks const fakeToken = "123:plainly-fake" //nolint:gosec // a fake bot token the test checks never leaks
// fakeBot serves one batch of updates, then nothing, and records the replies // fakeBot serves one batch of updates, then nothing, records the replies to
// to taps. // taps and the messages delivered, and refuses a message over Telegram's limit
// the way the Bot API does.
type fakeBot struct { type fakeBot struct {
mu sync.Mutex mu sync.Mutex
updates string updates string
served bool served bool
replies []string replies []string
sent []sentMessage
lastError string
// idle is called when the batch has been served and fetched past. // idle is called when the batch has been served and fetched past.
idle func() idle func()
} }
// sentMessage is one sendMessage the fake bot accepted.
type sentMessage struct {
Text string
Button bool
}
func (f *fakeBot) ServeHTTP(w http.ResponseWriter, r *http.Request) { func (f *fakeBot) ServeHTTP(w http.ResponseWriter, r *http.Request) {
f.mu.Lock() f.mu.Lock()
defer f.mu.Unlock() defer f.mu.Unlock()
@@ -37,6 +48,20 @@ func (f *fakeBot) ServeHTTP(w http.ResponseWriter, r *http.Request) {
f.idle() f.idle()
} }
_, _ = w.Write([]byte(`{"ok":true,"result":` + result + `}`)) _, _ = w.Write([]byte(`{"ok":true,"result":` + result + `}`))
case strings.HasSuffix(r.URL.Path, "/sendMessage"):
var body struct {
Text string `json:"text"`
ReplyMark map[string]any `json:"reply_markup"`
}
_ = json.UnmarshalRead(r.Body, &body)
if telegramtext.Units(body.Text) > telegramtext.MaxLen {
f.lastError = "sendMessage: Bad Request: message is too long"
_, _ = w.Write([]byte(`{"ok":false,"error_code":400,"description":"Bad Request: message is too long"}`))
return
}
f.sent = append(f.sent, sentMessage{Text: body.Text, Button: body.ReplyMark != nil})
id := len(f.sent)
_, _ = w.Write([]byte(`{"ok":true,"result":{"message_id":` + strconv.Itoa(id) + `}}`))
case strings.HasSuffix(r.URL.Path, "/answerCallbackQuery"): case strings.HasSuffix(r.URL.Path, "/answerCallbackQuery"):
var body struct { var body struct {
Text string `json:"text"` Text string `json:"text"`
@@ -107,6 +132,53 @@ func TestOnlyAnAllowedUsersTapIsAnAnswer(t *testing.T) {
} }
} }
// TestALongMessageGoesOutInParts guards the delivery of a long notification: the
// Bot API refuses one message over 4096 UTF-16 code units, and a delivery the
// host can never complete is retried for ever, so the body goes out as several
// messages in order, with the request's buttons on the last one.
func TestALongMessageGoesOutInParts(t *testing.T) {
body := strings.Repeat("the findings run to a length no single message holds. ", 300)
bot := &fakeBot{}
srv := httptest.NewServer(bot)
defer srv.Close()
tg, err := newTelegram(connector.Config{
Settings: map[string]string{"user_ids": "42"}, Secrets: map[string]string{"bot.token": fakeToken},
})
if err != nil {
t.Fatal(err)
}
tg.api = srv.URL
input, err := json.Marshal(map[string]string{"title": "Investigation finished", "body": body, "approval_request_id": "req-1"})
if err != nil {
t.Fatal(err)
}
if _, err := tg.Deliver(t.Context(), "message", input, "k"); err != nil {
t.Fatalf("deliver: %v (bot said %q)", err, bot.lastError)
}
if len(bot.sent) < 2 {
t.Fatalf("sent %d messages, want the body split over several", len(bot.sent))
}
for i, m := range bot.sent {
if units := telegramtext.Units(m.Text); units > telegramtext.MaxLen {
t.Errorf("message %d is %d units, over the limit", i+1, units)
}
if last := i == len(bot.sent)-1; m.Button != last {
t.Errorf("message %d buttons = %v, want them on the last only", i+1, m.Button)
}
}
if got := strings.Join(textsOf(bot.sent), ""); got != "Investigation finished\n\n"+body {
t.Errorf("the parts carry %d characters, want the whole message in order", len(got))
}
}
func textsOf(sent []sentMessage) []string {
out := make([]string, 0, len(sent))
for _, m := range sent {
out = append(out, m.Text)
}
return out
}
// TestAFailedCallNeverShowsTheToken guards the bot token: it travels in the // TestAFailedCallNeverShowsTheToken guards the bot token: it travels in the
// request URL, and a transport error must not carry that URL into a log. // request URL, and a transport error must not carry that URL into a log.
func TestAFailedCallNeverShowsTheToken(t *testing.T) { func TestAFailedCallNeverShowsTheToken(t *testing.T) {
+85
View File
@@ -0,0 +1,85 @@
// Package telegramtext holds the one rule both Telegram senders share: the Bot
// API accepts at most MaxLen UTF-16 code units in one message, so anything
// longer goes out as several messages, in order.
package telegramtext
import (
"unicode/utf8"
)
// MaxLen is Telegram's limit on one message's text, in UTF-16 code units. A
// rune outside the basic plane, an emoji for instance, counts as two.
const MaxLen = 4096
// Split cuts text into parts of at most max UTF-16 code units each, so the
// parts joined back together are the text it was given. It breaks on a blank
// line first, then on a line break, so a part ends where the author meant to
// pause, and never inside a character.
func Split(text string, max int) []string {
if max <= 0 {
max = MaxLen
}
if Units(text) <= max {
return []string{text}
}
var parts []string
for Units(text) > max {
cut := cutAt(text, max)
parts = append(parts, text[:cut])
text = text[cut:]
}
return append(parts, text)
}
// cutAt returns the byte index to cut s at, in preference order: after the last
// blank line that fits in max units, after the last line break that fits, or
// after the last character that fits.
func cutAt(s string, max int) int {
var (
used, after int
blank, line = -1, -1
)
for i, r := range s {
n := 1
if r > 0xFFFF {
n = 2
}
if used+n > max {
break
}
used += n
after = i + utf8.RuneLen(r)
if r != '\n' {
continue
}
if i > 0 && s[i-1] == '\n' {
blank = after
} else {
line = after
}
}
switch {
case blank > 0:
return blank
case line > 0:
return line
case after > 0:
return after
}
// max is too small to hold one character; cut it anyway rather than spin.
_, size := utf8.DecodeRuneInString(s)
return size
}
// Units is the length Telegram measures: UTF-16 code units, not runes.
func Units(s string) int {
n := 0
for _, r := range s {
if r > 0xFFFF {
n += 2
continue
}
n++
}
return n
}
+55
View File
@@ -0,0 +1,55 @@
package telegramtext
import (
"strings"
"testing"
"unicode/utf8"
)
// TestSplitKeepsEveryPartUnderTheLimit is the one behaviour both Telegram
// senders depend on: a long body goes out in parts, none of which the Bot API
// refuses, and together they still say everything.
func TestSplitKeepsEveryPartUnderTheLimit(t *testing.T) {
tests := []struct {
name string
text string
max int
parts int
// first is the part the splitter should have started with.
first string
}{
{name: "a message that fits is one part", text: "short", max: MaxLen, parts: 1},
{name: "a long paragraph is cut", text: strings.Repeat("a", 9000), max: 4000, parts: 3},
{name: "a blank line ends a part", text: "one\n\n" + strings.Repeat("a", 5000), max: 4000, parts: 3, first: "one\n\n"},
{name: "a line break is preferred to the middle of a word", text: strings.Repeat("hold a line\n", 500), max: 4000}, // An emoji is two UTF-16 units, so 3000 of them are 6000: cutting by
// runes alone would hand Telegram a message it rejects.
{name: "emoji count twice", text: strings.Repeat("😀", 3000), max: 4000, parts: 2},
{name: "an emoji is never cut in half", text: strings.Repeat("😀", 4001)},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
max := tt.max
if max == 0 {
max = MaxLen
}
parts := Split(tt.text, max)
if tt.parts > 0 && len(parts) != tt.parts {
t.Fatalf("Split gave %d parts, want %d", len(parts), tt.parts)
}
for i, p := range parts {
if units := Units(p); units > max {
t.Errorf("part %d is %d units, over the %d limit", i+1, units, max)
}
if !utf8.ValidString(p) {
t.Errorf("part %d cuts a character in half", i+1)
}
}
if tt.first != "" && parts[0] != tt.first {
t.Errorf("first part = %q, want %q", parts[0], tt.first)
}
if joined := strings.Join(parts, ""); joined != tt.text {
t.Errorf("the parts carry %d characters, want the %d they were given", len(joined), len(tt.text))
}
})
}
}