fix(telegram): a long message goes out in parts instead of failing for ever
publish / publish (push) Successful in 1m22s
publish / publish (push) Successful in 1m22s
The connector sent title and body as one sendMessage, so any notification over Telegram's 4096 UTF-16 unit limit was refused with 'message is too long' and, because a delivery is retried until the channel confirms it, retried every ten minutes for ever without ever arriving. The chat channel had its own splitter; both now share one in sdk/telegramtext, which counts UTF-16 units rather than runes so an emoji-heavy body isn't over the limit either.
This commit is contained in:
6 files changed
+228
-51
No files matched your search
@@ -22,6 +22,7 @@ import (
|
||||
|
||||
"github.com/samcharles93/archipelago/sdk/channel"
|
||||
gatewayv1 "github.com/samcharles93/archipelago/sdk/gateway/v1"
|
||||
"github.com/samcharles93/archipelago/sdk/telegramtext"
|
||||
)
|
||||
|
||||
const platform = "telegram"
|
||||
@@ -435,7 +436,7 @@ func helpText() string {
|
||||
|
||||
// sendText sends one message, split under Telegram's 4096-character limit.
|
||||
func (g *gateway) sendText(ctx context.Context, chatID int64, threadID int, text string) {
|
||||
for _, part := range splitText(text, messageMaxLen) {
|
||||
for _, part := range telegramtext.Split(text, messageMaxLen) {
|
||||
params := &bot.SendMessageParams{ChatID: chatID, Text: part}
|
||||
if threadID != 0 {
|
||||
params.MessageThreadID = threadID
|
||||
@@ -447,7 +448,7 @@ func (g *gateway) sendText(ctx context.Context, chatID int64, threadID int, text
|
||||
}
|
||||
|
||||
func (g *gateway) editText(ctx context.Context, chatID int64, messageID int, text string) {
|
||||
parts := splitText(text, messageMaxLen)
|
||||
parts := telegramtext.Split(text, messageMaxLen)
|
||||
if _, err := g.bot.EditMessageText(ctx, &bot.EditMessageTextParams{ChatID: chatID, MessageID: messageID, Text: parts[0]}); err != nil {
|
||||
g.log.DebugContext(ctx, "edit message failed", "chat_id", chatID, "error", err)
|
||||
}
|
||||
@@ -456,28 +457,6 @@ func (g *gateway) editText(ctx context.Context, chatID int64, messageID int, tex
|
||||
// messageMaxLen keeps one message under Telegram's 4096-character limit.
|
||||
const messageMaxLen = 4000
|
||||
|
||||
// splitText splits on paragraph boundaries, then hard-wraps long lines.
|
||||
func splitText(text string, max int) []string {
|
||||
if len([]rune(text)) <= max {
|
||||
return []string{text}
|
||||
}
|
||||
var parts []string
|
||||
for _, para := range strings.Split(text, "\n\n") {
|
||||
for len([]rune(para)) > max {
|
||||
cut := max
|
||||
if i := strings.LastIndex(para[:min(cut, len(para))], "\n"); i > 0 {
|
||||
cut = i
|
||||
}
|
||||
parts = append(parts, para[:cut])
|
||||
para = strings.TrimLeft(para[cut:], "\n")
|
||||
}
|
||||
if para != "" {
|
||||
parts = append(parts, para)
|
||||
}
|
||||
}
|
||||
return parts
|
||||
}
|
||||
|
||||
func (g *gateway) panicRecovery() bot.Middleware {
|
||||
return func(next bot.HandlerFunc) bot.HandlerFunc {
|
||||
return func(ctx context.Context, b *bot.Bot, update *models.Update) {
|
||||
|
||||
@@ -70,26 +70,6 @@ func TestParseSettings(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitText(t *testing.T) {
|
||||
long := strings.Repeat("a", 9000)
|
||||
parts := splitText(long, messageMaxLen)
|
||||
if len(parts) != 3 {
|
||||
t.Fatalf("splitText gave %d parts, want 3", len(parts))
|
||||
}
|
||||
for _, p := range parts {
|
||||
if len([]rune(p)) > messageMaxLen {
|
||||
t.Errorf("part is %d runes, over the limit", len([]rune(p)))
|
||||
}
|
||||
}
|
||||
if got := splitText("short", messageMaxLen); len(got) != 1 || got[0] != "short" {
|
||||
t.Errorf("splitText(short) = %v", got)
|
||||
}
|
||||
paras := splitText("one\n\n"+long, messageMaxLen)
|
||||
if paras[0] != "one" {
|
||||
t.Errorf("first part = %q, want %q", paras[0], "one")
|
||||
}
|
||||
}
|
||||
|
||||
func TestConversationKey(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
chatID, thread int64
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
|
||||
"github.com/samcharles93/archipelago/sdk/connector"
|
||||
connectorv1 "github.com/samcharles93/archipelago/sdk/connector/v1"
|
||||
"github.com/samcharles93/archipelago/sdk/telegramtext"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -84,8 +85,10 @@ type message struct {
|
||||
ApprovalRequestID string `json:"approval_request_id"`
|
||||
}
|
||||
|
||||
// Deliver writes the message to each allowed user. Telegram can't dedupe a
|
||||
// send, so a delivery fails only when no chat got it: retrying a partial one
|
||||
// Deliver writes the message to each allowed user. One message holds at most
|
||||
// 4096 UTF-16 code units, so a longer body goes out as several messages in
|
||||
// order, with an approval request's buttons on the last. Telegram can't dedupe
|
||||
// a send, so a delivery fails only when no chat got it: retrying a partial one
|
||||
// would repeat it to the users who already have it.
|
||||
func (t *telegram) Deliver(ctx context.Context, delivery string, input jsontext.Value, _ string) (string, error) {
|
||||
if delivery != "message" {
|
||||
@@ -101,9 +104,11 @@ func (t *telegram) Deliver(ctx context.Context, delivery string, input jsontext.
|
||||
}
|
||||
var ids []string
|
||||
var failed []error
|
||||
parts := telegramtext.Split(text, telegramtext.MaxLen)
|
||||
for _, chat := range t.users {
|
||||
body := map[string]any{"chat_id": chat, "text": text, "link_preview_options": map[string]any{"is_disabled": true}}
|
||||
if m.ApprovalRequestID != "" {
|
||||
for i, part := range parts {
|
||||
body := map[string]any{"chat_id": chat, "text": part, "link_preview_options": map[string]any{"is_disabled": true}}
|
||||
if i == len(parts)-1 && m.ApprovalRequestID != "" {
|
||||
body["reply_markup"] = map[string]any{"inline_keyboard": [][]map[string]any{{
|
||||
{"text": "Approve", "callback_data": answerPrefix + "yes:" + m.ApprovalRequestID},
|
||||
{"text": "Deny", "callback_data": answerPrefix + "no:" + m.ApprovalRequestID},
|
||||
@@ -114,10 +119,11 @@ func (t *telegram) Deliver(ctx context.Context, delivery string, input jsontext.
|
||||
}
|
||||
if err := t.call(ctx, "sendMessage", body, &sent); err != nil {
|
||||
failed = append(failed, fmt.Errorf("chat %d: %w", chat, err))
|
||||
continue
|
||||
break
|
||||
}
|
||||
ids = append(ids, strconv.FormatInt(chat, 10)+"/"+strconv.FormatInt(sent.MessageID, 10))
|
||||
}
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
return "", errors.Join(failed...)
|
||||
}
|
||||
|
||||
@@ -5,26 +5,37 @@ import (
|
||||
json "encoding/json/v2"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/samcharles93/archipelago/sdk/connector"
|
||||
"github.com/samcharles93/archipelago/sdk/telegramtext"
|
||||
)
|
||||
|
||||
const fakeToken = "123:plainly-fake" //nolint:gosec // a fake bot token the test checks never leaks
|
||||
|
||||
// fakeBot serves one batch of updates, then nothing, and records the replies
|
||||
// to taps.
|
||||
// fakeBot serves one batch of updates, then nothing, records the replies to
|
||||
// taps and the messages delivered, and refuses a message over Telegram's limit
|
||||
// the way the Bot API does.
|
||||
type fakeBot struct {
|
||||
mu sync.Mutex
|
||||
updates string
|
||||
served bool
|
||||
replies []string
|
||||
sent []sentMessage
|
||||
lastError string
|
||||
// idle is called when the batch has been served and fetched past.
|
||||
idle func()
|
||||
}
|
||||
|
||||
// sentMessage is one sendMessage the fake bot accepted.
|
||||
type sentMessage struct {
|
||||
Text string
|
||||
Button bool
|
||||
}
|
||||
|
||||
func (f *fakeBot) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
@@ -37,6 +48,20 @@ func (f *fakeBot) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
f.idle()
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"ok":true,"result":` + result + `}`))
|
||||
case strings.HasSuffix(r.URL.Path, "/sendMessage"):
|
||||
var body struct {
|
||||
Text string `json:"text"`
|
||||
ReplyMark map[string]any `json:"reply_markup"`
|
||||
}
|
||||
_ = json.UnmarshalRead(r.Body, &body)
|
||||
if telegramtext.Units(body.Text) > telegramtext.MaxLen {
|
||||
f.lastError = "sendMessage: Bad Request: message is too long"
|
||||
_, _ = w.Write([]byte(`{"ok":false,"error_code":400,"description":"Bad Request: message is too long"}`))
|
||||
return
|
||||
}
|
||||
f.sent = append(f.sent, sentMessage{Text: body.Text, Button: body.ReplyMark != nil})
|
||||
id := len(f.sent)
|
||||
_, _ = w.Write([]byte(`{"ok":true,"result":{"message_id":` + strconv.Itoa(id) + `}}`))
|
||||
case strings.HasSuffix(r.URL.Path, "/answerCallbackQuery"):
|
||||
var body struct {
|
||||
Text string `json:"text"`
|
||||
@@ -107,6 +132,53 @@ func TestOnlyAnAllowedUsersTapIsAnAnswer(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestALongMessageGoesOutInParts guards the delivery of a long notification: the
|
||||
// Bot API refuses one message over 4096 UTF-16 code units, and a delivery the
|
||||
// host can never complete is retried for ever, so the body goes out as several
|
||||
// messages in order, with the request's buttons on the last one.
|
||||
func TestALongMessageGoesOutInParts(t *testing.T) {
|
||||
body := strings.Repeat("the findings run to a length no single message holds. ", 300)
|
||||
bot := &fakeBot{}
|
||||
srv := httptest.NewServer(bot)
|
||||
defer srv.Close()
|
||||
tg, err := newTelegram(connector.Config{
|
||||
Settings: map[string]string{"user_ids": "42"}, Secrets: map[string]string{"bot.token": fakeToken},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tg.api = srv.URL
|
||||
input, err := json.Marshal(map[string]string{"title": "Investigation finished", "body": body, "approval_request_id": "req-1"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := tg.Deliver(t.Context(), "message", input, "k"); err != nil {
|
||||
t.Fatalf("deliver: %v (bot said %q)", err, bot.lastError)
|
||||
}
|
||||
if len(bot.sent) < 2 {
|
||||
t.Fatalf("sent %d messages, want the body split over several", len(bot.sent))
|
||||
}
|
||||
for i, m := range bot.sent {
|
||||
if units := telegramtext.Units(m.Text); units > telegramtext.MaxLen {
|
||||
t.Errorf("message %d is %d units, over the limit", i+1, units)
|
||||
}
|
||||
if last := i == len(bot.sent)-1; m.Button != last {
|
||||
t.Errorf("message %d buttons = %v, want them on the last only", i+1, m.Button)
|
||||
}
|
||||
}
|
||||
if got := strings.Join(textsOf(bot.sent), ""); got != "Investigation finished\n\n"+body {
|
||||
t.Errorf("the parts carry %d characters, want the whole message in order", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
func textsOf(sent []sentMessage) []string {
|
||||
out := make([]string, 0, len(sent))
|
||||
for _, m := range sent {
|
||||
out = append(out, m.Text)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// TestAFailedCallNeverShowsTheToken guards the bot token: it travels in the
|
||||
// request URL, and a transport error must not carry that URL into a log.
|
||||
func TestAFailedCallNeverShowsTheToken(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
// Package telegramtext holds the one rule both Telegram senders share: the Bot
|
||||
// API accepts at most MaxLen UTF-16 code units in one message, so anything
|
||||
// longer goes out as several messages, in order.
|
||||
package telegramtext
|
||||
|
||||
import (
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// MaxLen is Telegram's limit on one message's text, in UTF-16 code units. A
|
||||
// rune outside the basic plane, an emoji for instance, counts as two.
|
||||
const MaxLen = 4096
|
||||
|
||||
// Split cuts text into parts of at most max UTF-16 code units each, so the
|
||||
// parts joined back together are the text it was given. It breaks on a blank
|
||||
// line first, then on a line break, so a part ends where the author meant to
|
||||
// pause, and never inside a character.
|
||||
func Split(text string, max int) []string {
|
||||
if max <= 0 {
|
||||
max = MaxLen
|
||||
}
|
||||
if Units(text) <= max {
|
||||
return []string{text}
|
||||
}
|
||||
var parts []string
|
||||
for Units(text) > max {
|
||||
cut := cutAt(text, max)
|
||||
parts = append(parts, text[:cut])
|
||||
text = text[cut:]
|
||||
}
|
||||
return append(parts, text)
|
||||
}
|
||||
|
||||
// cutAt returns the byte index to cut s at, in preference order: after the last
|
||||
// blank line that fits in max units, after the last line break that fits, or
|
||||
// after the last character that fits.
|
||||
func cutAt(s string, max int) int {
|
||||
var (
|
||||
used, after int
|
||||
blank, line = -1, -1
|
||||
)
|
||||
for i, r := range s {
|
||||
n := 1
|
||||
if r > 0xFFFF {
|
||||
n = 2
|
||||
}
|
||||
if used+n > max {
|
||||
break
|
||||
}
|
||||
used += n
|
||||
after = i + utf8.RuneLen(r)
|
||||
if r != '\n' {
|
||||
continue
|
||||
}
|
||||
if i > 0 && s[i-1] == '\n' {
|
||||
blank = after
|
||||
} else {
|
||||
line = after
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case blank > 0:
|
||||
return blank
|
||||
case line > 0:
|
||||
return line
|
||||
case after > 0:
|
||||
return after
|
||||
}
|
||||
// max is too small to hold one character; cut it anyway rather than spin.
|
||||
_, size := utf8.DecodeRuneInString(s)
|
||||
return size
|
||||
}
|
||||
|
||||
// Units is the length Telegram measures: UTF-16 code units, not runes.
|
||||
func Units(s string) int {
|
||||
n := 0
|
||||
for _, r := range s {
|
||||
if r > 0xFFFF {
|
||||
n += 2
|
||||
continue
|
||||
}
|
||||
n++
|
||||
}
|
||||
return n
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package telegramtext
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// TestSplitKeepsEveryPartUnderTheLimit is the one behaviour both Telegram
|
||||
// senders depend on: a long body goes out in parts, none of which the Bot API
|
||||
// refuses, and together they still say everything.
|
||||
func TestSplitKeepsEveryPartUnderTheLimit(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
text string
|
||||
max int
|
||||
parts int
|
||||
// first is the part the splitter should have started with.
|
||||
first string
|
||||
}{
|
||||
{name: "a message that fits is one part", text: "short", max: MaxLen, parts: 1},
|
||||
{name: "a long paragraph is cut", text: strings.Repeat("a", 9000), max: 4000, parts: 3},
|
||||
{name: "a blank line ends a part", text: "one\n\n" + strings.Repeat("a", 5000), max: 4000, parts: 3, first: "one\n\n"},
|
||||
{name: "a line break is preferred to the middle of a word", text: strings.Repeat("hold a line\n", 500), max: 4000}, // An emoji is two UTF-16 units, so 3000 of them are 6000: cutting by
|
||||
// runes alone would hand Telegram a message it rejects.
|
||||
{name: "emoji count twice", text: strings.Repeat("😀", 3000), max: 4000, parts: 2},
|
||||
{name: "an emoji is never cut in half", text: strings.Repeat("😀", 4001)},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
max := tt.max
|
||||
if max == 0 {
|
||||
max = MaxLen
|
||||
}
|
||||
parts := Split(tt.text, max)
|
||||
if tt.parts > 0 && len(parts) != tt.parts {
|
||||
t.Fatalf("Split gave %d parts, want %d", len(parts), tt.parts)
|
||||
}
|
||||
for i, p := range parts {
|
||||
if units := Units(p); units > max {
|
||||
t.Errorf("part %d is %d units, over the %d limit", i+1, units, max)
|
||||
}
|
||||
if !utf8.ValidString(p) {
|
||||
t.Errorf("part %d cuts a character in half", i+1)
|
||||
}
|
||||
}
|
||||
if tt.first != "" && parts[0] != tt.first {
|
||||
t.Errorf("first part = %q, want %q", parts[0], tt.first)
|
||||
}
|
||||
if joined := strings.Join(parts, ""); joined != tt.text {
|
||||
t.Errorf("the parts carry %d characters, want the %d they were given", len(joined), len(tt.text))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user