fix(telegram): close a code fence only on a matching run
CommonMark fences are three OR MORE backticks or tildes, and the closing
fence must be at least as long as the opening one. The parser hardcoded a
three-backtick prefix for both ends, so every longer fence was mangled:
````go language became "`go", matching no grammar
```` wrapping ``` the outer fence closed at the inner one and the code
leaked out as paragraph blocks
~~~go never recognised, and the "~~" emphasis pass then ate
two of the three tildes
The middle case is the damaging one: wrapping a reply in a longer fence is
how a model safely emits content that itself contains a fence, so getting it
wrong could wreck an entire reply rather than just its formatting.
Track the opening fence's character and run length, and close only on a run
of the same character that is at least as long and carries no info string.
This commit is contained in:
1 parent
0b408017e0
commit
6d1be37e55
2 files changed
+161
-13
No files matched your search
@@ -227,12 +227,17 @@ func markdownToBlocks(md string) []models.InputRichBlock {
|
||||
// multi-line construct (paragraph, code fence, list, block quote) is
|
||||
// currently open.
|
||||
type markdownBlockParser struct {
|
||||
blocks []models.InputRichBlock
|
||||
paragraph strings.Builder
|
||||
codeLines []string
|
||||
codeLang string
|
||||
inCode bool
|
||||
indented bool
|
||||
blocks []models.InputRichBlock
|
||||
paragraph strings.Builder
|
||||
codeLines []string
|
||||
codeLang string
|
||||
inCode bool
|
||||
indented bool
|
||||
// fenceChar and fenceLen record the opening fence, so only a run of the
|
||||
// same character that is at least as long closes the block. Without
|
||||
// them a fence could never wrap content containing a shorter one.
|
||||
fenceChar byte
|
||||
fenceLen int
|
||||
listItems [][]models.InputRichBlock
|
||||
quoteLines []string
|
||||
// hardBreak records that the line just consumed ended in CommonMark's
|
||||
@@ -253,9 +258,10 @@ func (p *markdownBlockParser) handleLine(line string) {
|
||||
}
|
||||
|
||||
switch {
|
||||
case strings.HasPrefix(trimmed, "```"):
|
||||
case isFence(trimmed):
|
||||
char, length, info := parseFence(trimmed)
|
||||
p.flush()
|
||||
p.codeLang = strings.TrimSpace(strings.TrimPrefix(trimmed, "```"))
|
||||
p.codeLang, p.fenceChar, p.fenceLen = info, char, length
|
||||
p.inCode = true
|
||||
p.codeLines = nil
|
||||
case headingDepth(trimmed) > 0:
|
||||
@@ -327,13 +333,57 @@ func trimHardBreak(trimmed string) string {
|
||||
return strings.TrimRight(strings.TrimSuffix(trimmed, "\\"), " ")
|
||||
}
|
||||
|
||||
// fenceRun measures the leading run of CommonMark's two fence characters,
|
||||
// returning the character, its run length and the rest of the line. A run
|
||||
// shorter than three is not a fence, and is reported as length zero.
|
||||
func fenceRun(trimmed string) (byte, int, string) {
|
||||
if trimmed == "" || (trimmed[0] != '`' && trimmed[0] != '~') {
|
||||
return 0, 0, ""
|
||||
}
|
||||
char := trimmed[0]
|
||||
n := 0
|
||||
for n < len(trimmed) && trimmed[n] == char {
|
||||
n++
|
||||
}
|
||||
if n < 3 {
|
||||
return 0, 0, ""
|
||||
}
|
||||
return char, n, strings.TrimSpace(trimmed[n:])
|
||||
}
|
||||
|
||||
// isFence reports whether trimmed opens or closes a fenced code block.
|
||||
func isFence(trimmed string) bool {
|
||||
_, n, _ := fenceRun(trimmed)
|
||||
return n > 0
|
||||
}
|
||||
|
||||
// parseFence splits an opening fence into its character, run length and info
|
||||
// string. A backtick fence may not carry a backtick in its info string, which
|
||||
// is what keeps an inline code span from being read as a fence.
|
||||
func parseFence(trimmed string) (byte, int, string) {
|
||||
char, n, info := fenceRun(trimmed)
|
||||
if char == '`' && strings.Contains(info, "`") {
|
||||
return char, n, ""
|
||||
}
|
||||
return char, n, info
|
||||
}
|
||||
|
||||
// closesFence reports whether trimmed ends the open fenced block: the same
|
||||
// character, a run at least as long as the opening one, and nothing after it.
|
||||
// A shorter run, the other character, or a trailing info string is ordinary
|
||||
// content and stays in the block.
|
||||
func (p *markdownBlockParser) closesFence(trimmed string) bool {
|
||||
char, n, info := fenceRun(trimmed)
|
||||
return n > 0 && char == p.fenceChar && n >= p.fenceLen && info == ""
|
||||
}
|
||||
|
||||
// consumeCodeLine handles one line while a code block is open, reporting
|
||||
// whether that block claimed it. A fenced block ends at its closing fence; an
|
||||
// indented block ends at the first non-indented, non-blank line, which is not
|
||||
// a terminator and must still be parsed normally.
|
||||
func (p *markdownBlockParser) consumeCodeLine(line, trimmed string) bool {
|
||||
if !p.indented {
|
||||
if strings.HasPrefix(trimmed, "```") {
|
||||
if p.closesFence(trimmed) {
|
||||
p.closeCode()
|
||||
} else {
|
||||
p.codeLines = append(p.codeLines, line)
|
||||
@@ -450,6 +500,7 @@ func (p *markdownBlockParser) closeCode() {
|
||||
p.appendBlock(preformattedBlock(strings.Join(lines, "\n"), p.codeLang))
|
||||
}
|
||||
p.codeLines, p.codeLang, p.inCode, p.indented = nil, "", false, false
|
||||
p.fenceChar, p.fenceLen = 0, 0
|
||||
}
|
||||
|
||||
// indentedCodeLine reports whether line opens or continues an indented code
|
||||
|
||||
@@ -145,10 +145,10 @@ func TestMarkdownToBlocksStripsInlineMarkers(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarkdownToBlocksToolBlockBecomesParagraphAndCode(t *testing.T) {
|
||||
// The tool progress block composes a label line then a ```text fence.
|
||||
// Non-heading blocks get an explicit spacer paragraph on both sides; see
|
||||
// appendBlock.
|
||||
func TestMarkdownToBlocksLabelledFenceBecomesParagraphAndCode(t *testing.T) {
|
||||
// A label line followed by a fence. Non-heading blocks get an explicit
|
||||
// spacer paragraph on both sides; see appendBlock. That cost per fenced
|
||||
// block is why a one-line tool preview is not given one.
|
||||
in := "🔧 shell — done\n```text\nexit 0\n```\ndone"
|
||||
blocks := markdownToBlocks(in)
|
||||
if len(blocks) != 5 {
|
||||
@@ -461,3 +461,100 @@ func TestMarkdownToBlocksHardBreakDoesNotLeakPastABlankLine(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// blockLang returns a preformatted block's language, for assertions.
|
||||
func blockLang(b models.InputRichBlock) string {
|
||||
if b.Type == models.RichBlockTypePreformatted && b.InputRichBlockPreformatted != nil {
|
||||
return b.InputRichBlockPreformatted.Language
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// TestMarkdownToBlocksFenceRunLengths pins CommonMark's fence rules, which
|
||||
// models exercise constantly: a fence is three OR MORE of its character, and
|
||||
// the info string belongs to the opening fence alone.
|
||||
//
|
||||
// A hardcoded three-backtick prefix silently mangled every longer fence --
|
||||
// "````go" left one backtick glued to the language, so Telegram received
|
||||
// "`go" and matched no grammar at all.
|
||||
func TestMarkdownToBlocksFenceRunLengths(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
md string
|
||||
lang string
|
||||
text string
|
||||
}{
|
||||
{"three backticks", "```go\nx := 1\n```", "go", "x := 1"},
|
||||
{"four backticks", "````go\nx := 1\n````", "go", "x := 1"},
|
||||
{"six backticks", "``````go\nx := 1\n``````", "go", "x := 1"},
|
||||
{"three tildes", "~~~go\nx := 1\n~~~", "go", "x := 1"},
|
||||
{"five tildes", "~~~~~go\nx := 1\n~~~~~", "go", "x := 1"},
|
||||
{"no info string", "```\nx := 1\n```", "", "x := 1"},
|
||||
{"longer closer is allowed", "```go\nx := 1\n`````", "go", "x := 1"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
blocks := markdownToBlocks(tc.md)
|
||||
if len(blocks) != 1 {
|
||||
t.Fatalf("blocks = %d (%v), want 1 preformatted", len(blocks), blockTypes(blocks))
|
||||
}
|
||||
if got := blockLang(blocks[0]); got != tc.lang {
|
||||
t.Errorf("language = %q, want %q", got, tc.lang)
|
||||
}
|
||||
if got := blockText(blocks[0]); got != tc.text {
|
||||
t.Errorf("text = %q, want %q", got, tc.text)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestMarkdownToBlocksLongFenceWrapsShorterFence pins the reason longer
|
||||
// fences exist at all: wrapping content that itself contains a fence. The
|
||||
// inner fence must survive verbatim rather than closing the outer block and
|
||||
// spilling the code out as prose.
|
||||
func TestMarkdownToBlocksLongFenceWrapsShorterFence(t *testing.T) {
|
||||
blocks := markdownToBlocks("````\n```go\nx := 1\n```\n````")
|
||||
if len(blocks) != 1 {
|
||||
t.Fatalf("blocks = %d (%v), want 1 -- the inner fence must not close the outer one", len(blocks), blockTypes(blocks))
|
||||
}
|
||||
if got, want := blockText(blocks[0]), "```go\nx := 1\n```"; got != want {
|
||||
t.Errorf("text = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMarkdownToBlocksFenceClosesOnlyOnAMatchingRun pins the two ways a line
|
||||
// that looks like a fence is not one: too short, or the other character.
|
||||
func TestMarkdownToBlocksFenceClosesOnlyOnAMatchingRun(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
md string
|
||||
text string
|
||||
}{
|
||||
{"shorter run does not close", "````\na\n```\nb\n````", "a\n```\nb"},
|
||||
{"other character does not close", "```\na\n~~~\nb\n```", "a\n~~~\nb"},
|
||||
{"info string on closer does not close", "```go\na\n```go\nb\n```", "a\n```go\nb"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
blocks := markdownToBlocks(tc.md)
|
||||
if len(blocks) != 1 {
|
||||
t.Fatalf("blocks = %d (%v), want 1", len(blocks), blockTypes(blocks))
|
||||
}
|
||||
if got := blockText(blocks[0]); got != tc.text {
|
||||
t.Errorf("text = %q, want %q", got, tc.text)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestMarkdownToBlocksTildeFenceSurvivesTheEmphasisStripper pins that a tilde
|
||||
// fence is recognised before the strikethrough stripper reaches it. Falling
|
||||
// through to the paragraph case fed "~~~go" to the "~~" pass, which ate two
|
||||
// of the three tildes and rendered "~go".
|
||||
func TestMarkdownToBlocksTildeFenceSurvivesTheEmphasisStripper(t *testing.T) {
|
||||
blocks := markdownToBlocks("~~~\na ~~struck~~ b\n~~~")
|
||||
if len(blocks) != 1 || blocks[0].Type != models.RichBlockTypePreformatted {
|
||||
t.Fatalf("blocks = %v, want one preformatted block", blockTypes(blocks))
|
||||
}
|
||||
if got, want := blockText(blocks[0]), "a ~~struck~~ b"; got != want {
|
||||
t.Errorf("text = %q, want %q -- fenced content is verbatim", got, want)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user