Merge pull request #558 from samcharles93/feat/code-fence-aware-truncation
fix(telegram): preserve code fences across message splits
This commit is contained in:
commit
f4e79071c1
2 files changed
+287
-10
No files matched your search
@@ -934,26 +934,183 @@ func (g *Gateway) send(ctx context.Context, b *bot.Bot, chatID int64, messageThr
|
||||
}
|
||||
|
||||
// splitLongMessage divides text into parts of at most maxLen runes each,
|
||||
// preferring to cut immediately after a newline. Bounds are counted in
|
||||
// runes, not bytes, and every rune is retained, including blank lines and a
|
||||
// trailing newline. Preserving those boundaries matters for both Markdown
|
||||
// paragraphs and bare URLs that were separated from following prose.
|
||||
// preferring to cut immediately after a newline. A cut that would leave a
|
||||
// Markdown code fence open backs up to the boundary before that fence. Bounds
|
||||
// are counted in runes, not bytes, and every rune is retained, including blank
|
||||
// lines and a trailing newline. Preserving those boundaries matters for both
|
||||
// Markdown paragraphs and bare URLs that were separated from following prose.
|
||||
func splitLongMessage(text string, maxLen int) []string {
|
||||
text = normalizeTelegramText(text)
|
||||
if maxLen <= 0 || utf8.RuneCountInString(text) <= maxLen {
|
||||
return []string{text}
|
||||
}
|
||||
if hasUnsplitableFence(text, maxLen) {
|
||||
return splitFenceSyntaxAsPlainText(text, maxLen)
|
||||
}
|
||||
|
||||
runes := []rune(text)
|
||||
parts := make([]string, 0, (len(runes)+maxLen-1)/maxLen)
|
||||
reopen := ""
|
||||
for start := 0; start < len(runes); {
|
||||
part, end, nextReopen := nextMessagePart(runes, start, maxLen, reopen)
|
||||
parts = append(parts, part)
|
||||
start, reopen = end, nextReopen
|
||||
}
|
||||
return parts
|
||||
}
|
||||
|
||||
func nextMessagePart(runes []rune, start, maxLen int, reopen string) (string, int, string) {
|
||||
prefixLen := utf8.RuneCountInString(reopen)
|
||||
end := preferredMessageSplit(runes, start, maxLen-prefixLen)
|
||||
if end == len(runes) {
|
||||
return reopen + string(runes[start:end]), end, ""
|
||||
}
|
||||
|
||||
candidate := []rune(reopen + string(runes[start:end]))
|
||||
fenceStart, open := openCodeFenceStart(candidate, 0, len(candidate))
|
||||
if !open {
|
||||
return reopen + string(runes[start:end]), end, ""
|
||||
}
|
||||
if reopen == "" && fenceStart > 0 {
|
||||
end = start + fenceStart
|
||||
return string(runes[start:end]), end, ""
|
||||
}
|
||||
|
||||
delimiter := codeFenceDelimiter(candidate, fenceStart)
|
||||
opener := delimiter
|
||||
if reopen != "" {
|
||||
opener = strings.TrimSuffix(reopen, "\n")
|
||||
}
|
||||
suffix := "\n" + delimiter
|
||||
contentBudget := maxLen - prefixLen - utf8.RuneCountInString(suffix)
|
||||
if contentBudget <= 0 {
|
||||
return reopen + string(runes[start:end]), end, ""
|
||||
}
|
||||
end = min(start+contentBudget, len(runes))
|
||||
candidate = []rune(reopen + string(runes[start:end]))
|
||||
if _, stillOpen := openCodeFenceStart(candidate, 0, len(candidate)); !stillOpen {
|
||||
return reopen + string(runes[start:end]), end, ""
|
||||
}
|
||||
return reopen + string(runes[start:end]) + suffix, end, opener + "\n"
|
||||
}
|
||||
|
||||
func preferredMessageSplit(runes []rune, start, budget int) int {
|
||||
end := min(start+budget, len(runes))
|
||||
if end == len(runes) {
|
||||
return end
|
||||
}
|
||||
for i := end - 1; i > start; i-- {
|
||||
if runes[i] == '\n' {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return end
|
||||
}
|
||||
|
||||
func codeFenceDelimiter(runes []rune, start int) string {
|
||||
end := start
|
||||
for end < len(runes) && runes[end] != '\n' {
|
||||
end++
|
||||
}
|
||||
line := string(runes[start:end])
|
||||
markerStart := 0
|
||||
for markerStart < len(line) && markerStart < 3 && line[markerStart] == ' ' {
|
||||
markerStart++
|
||||
}
|
||||
markerEnd := markerStart
|
||||
for markerEnd < len(line) && line[markerEnd] == line[markerStart] {
|
||||
markerEnd++
|
||||
}
|
||||
return line[markerStart:markerEnd]
|
||||
}
|
||||
|
||||
// openCodeFenceStart reports the opening delimiter of a fenced Markdown block
|
||||
// that remains open at end. CommonMark permits up to three leading spaces and
|
||||
// uses runs of at least three backticks or tildes as fence delimiters.
|
||||
func openCodeFenceStart(runes []rune, start, end int) (int, bool) {
|
||||
openAt := -1
|
||||
marker := rune(0)
|
||||
width := 0
|
||||
for lineStart := start; lineStart < end; {
|
||||
lineEnd := lineStart
|
||||
for lineEnd < end && runes[lineEnd] != '\n' {
|
||||
lineEnd++
|
||||
}
|
||||
lineMarker, lineWidth, trailingBlank, fenced := codeFenceRun(runes, lineStart, lineEnd)
|
||||
if fenced && openAt < 0 && validCodeFenceOpening(runes, lineStart, lineEnd, lineMarker, lineWidth) {
|
||||
openAt, marker, width = lineStart, lineMarker, lineWidth
|
||||
} else if fenced && trailingBlank && lineMarker == marker && lineWidth >= width {
|
||||
openAt, marker, width = -1, 0, 0
|
||||
}
|
||||
lineStart = lineEnd + 1
|
||||
}
|
||||
return openAt, openAt >= 0
|
||||
}
|
||||
|
||||
func validCodeFenceOpening(runes []rune, lineStart, lineEnd int, marker rune, width int) bool {
|
||||
if marker != '`' {
|
||||
return true
|
||||
}
|
||||
markAt := lineStart
|
||||
for markAt < lineEnd && runes[markAt] == ' ' {
|
||||
markAt++
|
||||
}
|
||||
return !slices.Contains(runes[markAt+width:lineEnd], '`')
|
||||
}
|
||||
|
||||
func codeFenceRun(runes []rune, lineStart, lineEnd int) (rune, int, bool, bool) {
|
||||
markAt := lineStart
|
||||
for markAt < lineEnd && markAt-lineStart < 3 && runes[markAt] == ' ' {
|
||||
markAt++
|
||||
}
|
||||
if markAt == lineEnd || (runes[markAt] != '`' && runes[markAt] != '~') {
|
||||
return 0, 0, false, false
|
||||
}
|
||||
runEnd := markAt
|
||||
for runEnd < lineEnd && runes[runEnd] == runes[markAt] {
|
||||
runEnd++
|
||||
}
|
||||
width := runEnd - markAt
|
||||
trailingBlank := true
|
||||
for i := runEnd; i < lineEnd; i++ {
|
||||
if runes[i] != ' ' && runes[i] != '\t' {
|
||||
trailingBlank = false
|
||||
break
|
||||
}
|
||||
}
|
||||
return runes[markAt], width, trailingBlank, width >= 3
|
||||
}
|
||||
|
||||
// hasUnsplitableFence reports whether the bound is too small to close and
|
||||
// reopen a fence while consuming at least one source rune.
|
||||
func hasUnsplitableFence(text string, maxLen int) bool {
|
||||
runes := []rune(text)
|
||||
for lineStart := 0; lineStart < len(runes); {
|
||||
lineEnd := lineStart
|
||||
for lineEnd < len(runes) && runes[lineEnd] != '\n' {
|
||||
lineEnd++
|
||||
}
|
||||
_, width, _, fenced := codeFenceRun(runes, lineStart, lineEnd)
|
||||
if fenced && 2*width+2 >= maxLen {
|
||||
return true
|
||||
}
|
||||
lineStart = lineEnd + 1
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// splitFenceSyntaxAsPlainText preserves every source rune when a tiny bound
|
||||
// cannot carry valid fenced Markdown. Delimiter runs are divided across parts,
|
||||
// so no individual part presents an unterminated fence to the renderer.
|
||||
func splitFenceSyntaxAsPlainText(text string, maxLen int) []string {
|
||||
runes := []rune(text)
|
||||
parts := make([]string, 0, (len(runes)+maxLen-1)/maxLen)
|
||||
for start := 0; start < len(runes); {
|
||||
end := min(start+maxLen, len(runes))
|
||||
if end < len(runes) {
|
||||
for i := end - 1; i > start; i-- {
|
||||
if runes[i] == '\n' {
|
||||
end = i
|
||||
break
|
||||
}
|
||||
for i := start; i+2 < end; i++ {
|
||||
if (runes[i] == '`' || runes[i] == '~') && runes[i+1] == runes[i] && runes[i+2] == runes[i] {
|
||||
end = i + 2
|
||||
break
|
||||
}
|
||||
}
|
||||
parts = append(parts, string(runes[start:end]))
|
||||
|
||||
@@ -558,6 +558,126 @@ func TestSplitLongMessage(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestSplitLongMessageDoesNotCutInsideCodeFence(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
text string
|
||||
limit int
|
||||
parts int
|
||||
first string
|
||||
}{
|
||||
{
|
||||
name: "unfenced text over limit uses ordinary boundary",
|
||||
text: "first paragraph\nsecond paragraph",
|
||||
limit: 20,
|
||||
parts: 2,
|
||||
first: "first paragraph",
|
||||
},
|
||||
{
|
||||
name: "fenced text exactly at limit is unchanged",
|
||||
text: "```go\nhello\n```",
|
||||
limit: 15,
|
||||
parts: 1,
|
||||
first: "```go\nhello\n```",
|
||||
},
|
||||
{
|
||||
name: "cut inside fence backs up before opening fence",
|
||||
text: "intro\n\n```go\nhello\n```\noutro",
|
||||
limit: 18,
|
||||
parts: 3,
|
||||
first: "intro\n\n",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
parts := splitLongMessage(tt.text, tt.limit)
|
||||
if len(parts) != tt.parts {
|
||||
t.Fatalf("parts = %q, want %d parts", parts, tt.parts)
|
||||
}
|
||||
if parts[0] != tt.first {
|
||||
t.Errorf("first part = %q, want %q", parts[0], tt.first)
|
||||
}
|
||||
for i, part := range parts {
|
||||
if strings.Count(part, "```")%2 != 0 {
|
||||
t.Errorf("part %d cuts an open code fence: %q", i, part)
|
||||
}
|
||||
if got := utf8.RuneCountInString(part); got > tt.limit {
|
||||
t.Errorf("part %d has %d runes, want at most %d", i, got, tt.limit)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("fence longer than one message is closed and reopened", func(t *testing.T) {
|
||||
const limit = 20
|
||||
text := "```go\n" + strings.Repeat("x", limit*2) + "\n```"
|
||||
parts := splitLongMessage(text, limit)
|
||||
if len(parts) < 3 {
|
||||
t.Fatalf("parts = %q, want at least three bounded parts", parts)
|
||||
}
|
||||
for i, part := range parts {
|
||||
if strings.Count(part, "```")%2 != 0 {
|
||||
t.Errorf("part %d cuts an open code fence: %q", i, part)
|
||||
}
|
||||
if got := utf8.RuneCountInString(part); got > limit {
|
||||
t.Errorf("part %d has %d runes, want at most %d", i, got, limit)
|
||||
}
|
||||
}
|
||||
if got := strings.Count(strings.Join(parts, ""), "x"); got != limit*2 {
|
||||
t.Errorf("split retained %d code runes, want %d", got, limit*2)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("long fence info string cannot exceed continuation budget", func(t *testing.T) {
|
||||
const limit = 20
|
||||
text := "```" + strings.Repeat("language", limit) + "\nbody\n```"
|
||||
for i, part := range splitLongMessage(text, limit) {
|
||||
if got := utf8.RuneCountInString(part); got > limit {
|
||||
t.Errorf("part %d has %d runes, want at most %d", i, got, limit)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("limits too small for fence continuation preserve bounded plain text", func(t *testing.T) {
|
||||
const text = "```important\nabc\n```"
|
||||
for _, limit := range []int{3, 7, 8} {
|
||||
parts := splitLongMessage(text, limit)
|
||||
for i, part := range parts {
|
||||
if strings.Contains(part, "```") {
|
||||
t.Errorf("limit %d part %d retained an unsplitable fence: %q", limit, i, part)
|
||||
}
|
||||
if got := utf8.RuneCountInString(part); got > limit {
|
||||
t.Errorf("limit %d part %d has %d runes, want at most %d", limit, i, got, limit)
|
||||
}
|
||||
}
|
||||
if got := strings.Join(parts, ""); got != text {
|
||||
t.Errorf("limit %d rebuilt = %q, want every source rune retained", limit, got)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("fence-like code line with trailing text is not a closer", func(t *testing.T) {
|
||||
const limit = 24
|
||||
text := "```go\nfirst\n```not closed\nsecond\n```\nafter"
|
||||
parts := splitLongMessage(text, limit)
|
||||
for i, part := range parts {
|
||||
partRunes := []rune(part)
|
||||
if _, open := openCodeFenceStart(partRunes, 0, len(partRunes)); open {
|
||||
t.Errorf("part %d cuts an open code fence: %q", i, part)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("backtick in info string is not a code fence", func(t *testing.T) {
|
||||
const text = "```foo`bar\nplain text that crosses the limit"
|
||||
parts := splitLongMessage(text, 18)
|
||||
if got := strings.Join(parts, ""); got != text {
|
||||
t.Errorf("rebuilt = %q, want invalid fence syntax treated as plain text", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestIsSenderAllowed(t *testing.T) {
|
||||
g := New("tok", "", "", []int64{100, 200, 300}, slog.Default())
|
||||
|
||||
|
||||
Reference in new issue
Block a user