mirror of
https://github.com/HKUDS/nanobot.git
synced 2026-08-31 08:13:11 +03:00
fix(tui): bound Markdown math preprocessing
This commit is contained in:
@@ -38,12 +38,16 @@ describe("terminal LaTeX rendering", () => {
|
|||||||
const source = [
|
const source = [
|
||||||
"Costs are $24 today or $10-20 later; variables $x$ and $2^n$ are math.",
|
"Costs are $24 today or $10-20 later; variables $x$ and $2^n$ are math.",
|
||||||
"Shipping costs $5+$10 and paths use $HOME/$USER.",
|
"Shipping costs $5+$10 and paths use $HOME/$USER.",
|
||||||
|
"Mixed currency and variables like $5+$x$ remain literal.",
|
||||||
|
"Commands and groups stay literal: $5+$\\alpha$, $5+${x}$, and $5+$(x)$.",
|
||||||
"Matrix $A$ maps $V$ to $W$.",
|
"Matrix $A$ maps $V$ to $W$.",
|
||||||
].join("\n")
|
].join("\n")
|
||||||
|
|
||||||
expect(renderLatexAsUnicode(source)).toBe([
|
expect(renderLatexAsUnicode(source)).toBe([
|
||||||
"Costs are $24 today or $10-20 later; variables x and 2ⁿ are math.",
|
"Costs are $24 today or $10-20 later; variables x and 2ⁿ are math.",
|
||||||
"Shipping costs $5+$10 and paths use $HOME/$USER.",
|
"Shipping costs $5+$10 and paths use $HOME/$USER.",
|
||||||
|
"Mixed currency and variables like $5+$x$ remain literal.",
|
||||||
|
"Commands and groups stay literal: $5+$\\alpha$, $5+${x}$, and $5+$(x)$.",
|
||||||
"Matrix A maps V to W.",
|
"Matrix A maps V to W.",
|
||||||
].join("\n"))
|
].join("\n"))
|
||||||
})
|
})
|
||||||
@@ -70,6 +74,7 @@ describe("terminal LaTeX rendering", () => {
|
|||||||
"\\[\\frac{a}{b}\\]",
|
"\\[\\frac{a}{b}\\]",
|
||||||
"```",
|
"```",
|
||||||
].join("\n"))
|
].join("\n"))
|
||||||
|
expect(renderLatexAsUnicode("`$x$``")).toBe("`x``")
|
||||||
})
|
})
|
||||||
|
|
||||||
test("leaves fenced code inside block quotes and lists unchanged", () => {
|
test("leaves fenced code inside block quotes and lists unchanged", () => {
|
||||||
@@ -101,9 +106,19 @@ describe("terminal LaTeX rendering", () => {
|
|||||||
test("handles many unmatched openers without changing them", () => {
|
test("handles many unmatched openers without changing them", () => {
|
||||||
const openers = "\\(".repeat(20_000)
|
const openers = "\\(".repeat(20_000)
|
||||||
const backslashes = "\\".repeat(20_000)
|
const backslashes = "\\".repeat(20_000)
|
||||||
|
const backticks = `text ${"`".repeat(20_000)} then $x$`
|
||||||
|
const descendingBackticks = "text " + Array.from(
|
||||||
|
{ length: 500 },
|
||||||
|
(_, index) => "`".repeat(500 - index),
|
||||||
|
).join("x") + " $x$"
|
||||||
|
const linkDestinations = `${"](".repeat(20_000)} then $x$`
|
||||||
|
|
||||||
expect(renderLatexAsUnicode(openers)).toBe(openers)
|
expect(renderLatexAsUnicode(openers)).toBe(openers)
|
||||||
expect(renderLatexAsUnicode(backslashes)).toBe(backslashes)
|
expect(renderLatexAsUnicode(backslashes)).toBe(backslashes)
|
||||||
|
expect(renderLatexAsUnicode(backticks)).toBe(backticks.replace("$x$", "x"))
|
||||||
|
expect(renderLatexAsUnicode(descendingBackticks))
|
||||||
|
.toBe(descendingBackticks.replace("$x$", "x"))
|
||||||
|
expect(renderLatexAsUnicode(linkDestinations)).toBe(linkDestinations.replace("$x$", "x"))
|
||||||
})
|
})
|
||||||
|
|
||||||
test("leaves unsupported, malformed, and deeply nested math unchanged", () => {
|
test("leaves unsupported, malformed, and deeply nested math unchanged", () => {
|
||||||
|
|||||||
+81
-24
@@ -263,9 +263,16 @@ const MAX_GROUP_DEPTH = 64
|
|||||||
|
|
||||||
/** Converts supported LaTeX math spans without changing the stored Markdown source. */
|
/** Converts supported LaTeX math spans without changing the stored Markdown source. */
|
||||||
export function renderLatexAsUnicode(markdown: string): string {
|
export function renderLatexAsUnicode(markdown: string): string {
|
||||||
|
if (!markdown.includes("$") && !markdown.includes("\\(") && !markdown.includes("\\[")) {
|
||||||
|
return markdown
|
||||||
|
}
|
||||||
let rendered = ""
|
let rendered = ""
|
||||||
let cursor = 0
|
let cursor = 0
|
||||||
const missingClosers = new Set<string>()
|
const missingClosers = new Set<string>()
|
||||||
|
const inlineCodeEnds = markdown.includes("`") ? markdownInlineCodeEnds(markdown) : null
|
||||||
|
const linkDestinationEnds = markdown.includes("](")
|
||||||
|
? markdownLinkDestinationEnds(markdown)
|
||||||
|
: null
|
||||||
|
|
||||||
while (cursor < markdown.length) {
|
while (cursor < markdown.length) {
|
||||||
const fencedEnd = fencedCodeEnd(markdown, cursor)
|
const fencedEnd = fencedCodeEnd(markdown, cursor)
|
||||||
@@ -276,15 +283,19 @@ export function renderLatexAsUnicode(markdown: string): string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (markdown[cursor] === "`") {
|
if (markdown[cursor] === "`") {
|
||||||
const inlineEnd = inlineCodeEnd(markdown, cursor)
|
const delimiterLength = repeatedCharacterCount(markdown, cursor, "`")
|
||||||
if (inlineEnd !== null) {
|
const inlineEnd = inlineCodeEnds?.get(cursor)
|
||||||
|
if (inlineEnd !== undefined) {
|
||||||
rendered += markdown.slice(cursor, inlineEnd)
|
rendered += markdown.slice(cursor, inlineEnd)
|
||||||
cursor = inlineEnd
|
cursor = inlineEnd
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
rendered += markdown.slice(cursor, cursor + delimiterLength)
|
||||||
|
cursor += delimiterLength
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
const destinationEnd = markdownDestinationEnd(markdown, cursor)
|
const destinationEnd = markdownDestinationEnd(markdown, cursor, linkDestinationEnds)
|
||||||
if (destinationEnd !== null) {
|
if (destinationEnd !== null) {
|
||||||
rendered += markdown.slice(cursor, destinationEnd)
|
rendered += markdown.slice(cursor, destinationEnd)
|
||||||
cursor = destinationEnd
|
cursor = destinationEnd
|
||||||
@@ -294,6 +305,11 @@ export function renderLatexAsUnicode(markdown: string): string {
|
|||||||
const explicitOpening = explicitMathOpeningAt(markdown, cursor)
|
const explicitOpening = explicitMathOpeningAt(markdown, cursor)
|
||||||
const math = mathSpanAt(markdown, cursor, explicitOpening, missingClosers)
|
const math = mathSpanAt(markdown, cursor, explicitOpening, missingClosers)
|
||||||
if (math) {
|
if (math) {
|
||||||
|
if (math.literal) {
|
||||||
|
rendered += markdown.slice(cursor, math.end)
|
||||||
|
cursor = math.end
|
||||||
|
continue
|
||||||
|
}
|
||||||
const converted = convertMath(math.content)
|
const converted = convertMath(math.content)
|
||||||
if (converted !== null) {
|
if (converted !== null) {
|
||||||
rendered += escapeMarkdown(converted)
|
rendered += escapeMarkdown(converted)
|
||||||
@@ -320,6 +336,7 @@ export function renderLatexAsUnicode(markdown: string): string {
|
|||||||
interface MathSpan {
|
interface MathSpan {
|
||||||
content: string
|
content: string
|
||||||
end: number
|
end: number
|
||||||
|
literal?: boolean
|
||||||
}
|
}
|
||||||
|
|
||||||
interface ParseState {
|
interface ParseState {
|
||||||
@@ -389,7 +406,23 @@ function mathSpanAt(
|
|||||||
const span = delimitedMath(markdown, start, "$", "$", true, missingClosers)
|
const span = delimitedMath(markdown, start, "$", "$", true, missingClosers)
|
||||||
if (!span || isWhitespace(span.content.at(-1))) return null
|
if (!span || isWhitespace(span.content.at(-1))) return null
|
||||||
const first = span.content[0]
|
const first = span.content[0]
|
||||||
if (first && /[0-9]/u.test(first) && /[0-9]/u.test(markdown[span.end] ?? "")) return null
|
if (first && /[0-9]/u.test(first)) {
|
||||||
|
const following = markdown[span.end] ?? ""
|
||||||
|
if (!isWhitespace(following) && /[+\-*/]\s*$/u.test(span.content)) {
|
||||||
|
const adjacent = delimitedMath(
|
||||||
|
markdown,
|
||||||
|
span.end - 1,
|
||||||
|
"$",
|
||||||
|
"$",
|
||||||
|
true,
|
||||||
|
missingClosers,
|
||||||
|
)
|
||||||
|
if (adjacent && !isWhitespace(adjacent.content.at(-1))) {
|
||||||
|
return { content: "", end: adjacent.end, literal: true }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (/[0-9]/u.test(following)) return null
|
||||||
|
}
|
||||||
if (first && /[0-9]/u.test(first) && !/[\\^_={}|+<>]/u.test(span.content)) return null
|
if (first && /[0-9]/u.test(first) && !/[\\^_={}|+<>]/u.test(span.content)) return null
|
||||||
if (
|
if (
|
||||||
/^[A-Z_][A-Z0-9_]*(?:\/|\s.*)$/u.test(span.content)
|
/^[A-Z_][A-Z0-9_]*(?:\/|\s.*)$/u.test(span.content)
|
||||||
@@ -491,32 +524,56 @@ function closingFenceOffset(line: string, context: FenceContext): number | null
|
|||||||
return indent === undefined ? null : offset + indent.length
|
return indent === undefined ? null : offset + indent.length
|
||||||
}
|
}
|
||||||
|
|
||||||
function inlineCodeEnd(source: string, start: number): number | null {
|
function markdownInlineCodeEnds(source: string): ReadonlyMap<number, number> {
|
||||||
const count = repeatedCharacterCount(source, start, "`")
|
const ends = new Map<number, number>()
|
||||||
const delimiter = "`".repeat(count)
|
const previousRun = new Map<number, number>()
|
||||||
const close = source.indexOf(delimiter, start + count)
|
let cursor = 0
|
||||||
return close < 0 ? null : close + count
|
|
||||||
}
|
|
||||||
|
|
||||||
function markdownDestinationEnd(source: string, start: number): number | null {
|
|
||||||
if (source[start] === "(" && source[start - 1] === "]" && !isEscaped(source, start - 1)) {
|
|
||||||
let depth = 1
|
|
||||||
let cursor = start + 1
|
|
||||||
while (cursor < source.length) {
|
while (cursor < source.length) {
|
||||||
const value = source[cursor]
|
if (source[cursor] !== "`") {
|
||||||
if (value === "\n") return null
|
cursor += 1
|
||||||
if (value === "\\") {
|
|
||||||
cursor += 2
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if (value === "(") depth += 1
|
const count = repeatedCharacterCount(source, cursor, "`")
|
||||||
else if (value === ")") {
|
const previous = previousRun.get(count)
|
||||||
depth -= 1
|
if (previous !== undefined) ends.set(previous, cursor + count)
|
||||||
if (depth === 0) return cursor + 1
|
previousRun.set(count, cursor)
|
||||||
|
cursor += count
|
||||||
|
}
|
||||||
|
return ends
|
||||||
|
}
|
||||||
|
|
||||||
|
function markdownLinkDestinationEnds(source: string): ReadonlyMap<number, number> {
|
||||||
|
const ends = new Map<number, number>()
|
||||||
|
const openParentheses: number[] = []
|
||||||
|
let cursor = 0
|
||||||
|
while (cursor < source.length) {
|
||||||
|
const value = source[cursor]
|
||||||
|
if (value === "\\") {
|
||||||
|
cursor += Math.min(2, source.length - cursor)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if (value === "\n") {
|
||||||
|
openParentheses.length = 0
|
||||||
|
} else if (value === "(") {
|
||||||
|
openParentheses.push(cursor)
|
||||||
|
} else if (value === ")") {
|
||||||
|
const opening = openParentheses.pop()
|
||||||
|
if (opening !== undefined && source[opening - 1] === "]" && !isEscaped(source, opening - 1)) {
|
||||||
|
ends.set(opening, cursor + 1)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
cursor += 1
|
cursor += 1
|
||||||
}
|
}
|
||||||
return null
|
return ends
|
||||||
|
}
|
||||||
|
|
||||||
|
function markdownDestinationEnd(
|
||||||
|
source: string,
|
||||||
|
start: number,
|
||||||
|
linkDestinationEnds: ReadonlyMap<number, number> | null,
|
||||||
|
): number | null {
|
||||||
|
if (source[start] === "(" && source[start - 1] === "]" && !isEscaped(source, start - 1)) {
|
||||||
|
return linkDestinationEnds?.get(start) ?? null
|
||||||
}
|
}
|
||||||
|
|
||||||
if (source[start] === "<") {
|
if (source[start] === "<") {
|
||||||
|
|||||||
Reference in New Issue
Block a user