diff --git a/packages/tui/src/components/markdown.ts b/packages/tui/src/components/markdown.ts index a3b526d73..bacfe29f8 100644 --- a/packages/tui/src/components/markdown.ts +++ b/packages/tui/src/components/markdown.ts @@ -3,11 +3,21 @@ import markedCjkFriendly from "marked-cjk-friendly"; import { renderLatex } from "../latex.ts"; import { getCapabilities, hyperlink, isImageLine } from "../terminal-image.ts"; import type { Component } from "../tui.ts"; -import { applyBackgroundToLine, flattenLines, visibleWidth, wrapTextWithAnsi } from "../utils.ts"; +import { applyBackgroundToLine, cjkPunctuationRegex, flattenLines, visibleWidth, wrapTextWithAnsi } from "../utils.ts"; const STRICT_STRIKETHROUGH_REGEX = /^(~~)(?=[^\s~])((?:\\.|[^\\])*?(?:\\.|[^\s~\\]))\1(?=[^~]|$)/; -class StrictStrikethroughTokenizer extends Tokenizer { +class MarkdownTokenizer extends Tokenizer { + override url(src: string): Tokens.Link | undefined { + const token = super.url(src); + if (!token) return undefined; + + // Bare URLs stop at CJK punctuation; explicit links and angle autolinks + // use separate tokenizers and can still contain these characters. + const punctuationIndex = token.raw.search(cjkPunctuationRegex); + return punctuationIndex < 0 ? token : super.url(token.raw.slice(0, punctuationIndex)); + } + override del(src: string): Tokens.Del | undefined { const match = STRICT_STRIKETHROUGH_REGEX.exec(src); if (!match) { @@ -189,7 +199,7 @@ function trimPartialClosingFences(tokens: readonly Token[]): void { const markdownParser = new Marked(); markdownParser.setOptions({ - tokenizer: new StrictStrikethroughTokenizer(), + tokenizer: new MarkdownTokenizer(), }); markdownParser.use({ tokenizer: { emStrong: markedCjkFriendly().tokenizer?.emStrong }, diff --git a/packages/tui/test/markdown.test.ts b/packages/tui/test/markdown.test.ts index b20d84dfe..0cdcf172b 100644 --- a/packages/tui/test/markdown.test.ts +++ b/packages/tui/test/markdown.test.ts @@ -1716,6 +1716,49 @@ bar`, resetCapabilitiesCache(); }); + it("should stop bare URLs at CJK punctuation and preserve the following text", () => { + setCapabilities({ images: null, trueColor: false, hyperlinks: true }); + for (const punctuation of "。,;:!?、“”‘’()【】《》「」『』…—·・〈〉[]{}.。「」、・") { + for (const url of ["https://example.com", "https://example.com/中文?q=搜索#片段", "www.example.com"]) { + const source = `${url}${punctuation}后面的中文`; + const output = new Markdown(source, 0, 0, defaultMarkdownTheme).render(160).join(""); + const href = url.startsWith("www.") ? `http://${url}` : url; + assert.ok(output.includes(`\x1b]8;;${href}\x1b\\`), source); + assert.ok(stripAnsi(output).includes(`\x1b]8;;\x1b\\${punctuation}后面的中文`), source); + assert.strictEqual(stripAnsi(output.replace(/\x1b\]8;;[^\x1b]*\x1b\\/g, "")).trimEnd(), source); + } + } + }); + + it("should preserve Chinese characters without punctuation in bare URLs", () => { + setCapabilities({ images: null, trueColor: false, hyperlinks: true }); + for (const url of [ + "https://example.com紧接着中文", + "https://example.com/中文?q=搜索#片段", + "https://example.com/テストー한글", + ]) { + const output = new Markdown(url, 0, 0, defaultMarkdownTheme).render(160).join(""); + assert.ok(output.includes(`\x1b]8;;${url}\x1b\\`)); + } + }); + + it("should preserve CJK punctuation in explicit links and angle autolinks", () => { + setCapabilities({ images: null, trueColor: false, hyperlinks: true }); + const url = "https://example.com/中文,路径·テスト・한글…—。"; + for (const source of [`[文档](${url})`, `<${url}>`]) { + const output = new Markdown(source, 0, 0, defaultMarkdownTheme).render(160).join(""); + assert.ok(output.includes(`\x1b]8;;${url}\x1b\\`), source); + } + }); + + it("should preserve English punctuation trimming before Chinese punctuation", () => { + setCapabilities({ images: null, trueColor: false, hyperlinks: true }); + const source = "https://example.com.,后面的中文"; + const output = new Markdown(source, 0, 0, defaultMarkdownTheme).render(160).join(""); + assert.ok(output.includes("\x1b]8;;https://example.com\x1b\\")); + assert.ok(stripAnsi(output).includes("\x1b]8;;\x1b\\.,后面的中文")); + }); + it("should not duplicate URL for autolinked emails", () => { // Hyperlinks capability does not affect the mailto: display check. setCapabilities({ images: null, trueColor: false, hyperlinks: false });