diff --git a/bun.lock b/bun.lock index 1c67ac3a5..30da0ed63 100644 --- a/bun.lock +++ b/bun.lock @@ -15,6 +15,7 @@ "dependencies": { "bun-ffi-structs": "0.2.4", "diff": "9.0.0", + "entities": "7.0.1", "marked": "17.0.1", "string-width": "7.2.0", "strip-ansi": "7.1.2", @@ -489,6 +490,22 @@ "@opentui/core": ["@opentui/core@workspace:packages/core"], + "@opentui/core-darwin-arm64": ["@opentui/core-darwin-arm64@0.4.5", "", { "os": "darwin", "cpu": "arm64" }, "sha512-8KUG0oRidnR+oW1RSZJ72/PhZLl+qRRMk5U/mieF4c0SJ5V3tYACpBZAKzQfHNd1f7QzD8FHZct1lPpQgtmkWg=="], + + "@opentui/core-darwin-x64": ["@opentui/core-darwin-x64@0.4.5", "", { "os": "darwin", "cpu": "x64" }, "sha512-R2bocsg55gwjOqCp/MWFgFYzRmsduKegB6nzgFAPCvAD/L5Jf30xpWJWFlSg3x8vxe1L9WJ84dfqa4M7mZZ3wA=="], + + "@opentui/core-linux-arm64": ["@opentui/core-linux-arm64@0.4.5", "", { "os": "linux", "cpu": "arm64" }, "sha512-R4MZ25a4CzOAGVjW9aj1hUfzQGVfCJwrwBDbNs2SXaIvzcZqkxCVtU4FoQ5LsaD0j/BdNQVg2CIfFkFsm1fDuQ=="], + + "@opentui/core-linux-arm64-musl": ["@opentui/core-linux-arm64-musl@0.4.5", "", { "os": "linux", "cpu": "arm64" }, "sha512-ieqdyKI6EIYPalYAETB2wsdP83hr5Ifi+dFnBFUmdEEFHsoKwBmn2S7bsTOYlX7Bg03F4/YPIg+IvRpeC+cUJw=="], + + "@opentui/core-linux-x64": ["@opentui/core-linux-x64@0.4.5", "", { "os": "linux", "cpu": "x64" }, "sha512-SNyuQoxMKI1vuJhgxSSW96adWM6LqFl2SoS3GM4tGeneGOanVVG2Y06PvlytXvF4cKik97t0rqkVMRetmOs93w=="], + + "@opentui/core-linux-x64-musl": ["@opentui/core-linux-x64-musl@0.4.5", "", { "os": "linux", "cpu": "x64" }, "sha512-mKVKcIcPiSVVZZsdPSBoWwoa2/TCeQAaMDeHF7PFw2kt5bTXZPP7xxWfRQLCNIcA1eaGl59UuwUWHDR2Ve548Q=="], + + "@opentui/core-win32-arm64": ["@opentui/core-win32-arm64@0.4.5", "", { "os": "win32", "cpu": "arm64" }, "sha512-GHTTsqeR45q2Iek9Rb7ty+x/hAKn2jZ1ujlCgPR8LBKyF7h0E1dNFryoZ7ehMc3kJndP1sKn836IemKFqxuDdQ=="], + + "@opentui/core-win32-x64": ["@opentui/core-win32-x64@0.4.5", "", { "os": "win32", "cpu": "x64" }, "sha512-Y8T/yXCDGagRGiQrtmuB6AhRcPucKFs/Dre3v8kJwNYqDccI4FzUPKclZ7djfmRZNjl7JUqPhZZP/PwDpQocMg=="], + "@opentui/examples": ["@opentui/examples@workspace:packages/examples"], "@opentui/keymap": ["@opentui/keymap@workspace:packages/keymap"], diff --git a/packages/core/package.json b/packages/core/package.json index 449ff7ebe..f6010c2af 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -32,6 +32,7 @@ "bench:render-runtimes": "bun src/benchmark/render-runtime-benchmark.ts", "bench:render-compare": "bun src/benchmark/render-runtime-compare.ts", "bench:text-table": "bun src/benchmark/text-table-benchmark.ts", + "bench:markdown-links": "bun src/benchmark/markdown-link-benchmark.ts", "bench:text-table-width": "bun src/benchmark/text-table-width-benchmark.ts", "bench:ts": "bun src/benchmark/native-span-feed-benchmark.ts --suite=quick --json=src/benchmark/latest-quick-bench-run.json && bun src/benchmark/native-span-feed-benchmark.ts --suite=default --json=src/benchmark/latest-default-bench-run.json && bun src/benchmark/native-span-feed-benchmark.ts --suite=large --json=src/benchmark/latest-large-bench-run.json && bun src/benchmark/native-span-feed-benchmark.ts --suite=all --json=src/benchmark/latest-all-bench-run.json && bun src/benchmark/native-span-feed-async-benchmark.ts --json=src/benchmark/latest-async-bench-run.json", "publish": "bun scripts/publish.ts", @@ -53,6 +54,7 @@ "dependencies": { "bun-ffi-structs": "0.2.4", "diff": "9.0.0", + "entities": "7.0.1", "marked": "17.0.1", "string-width": "7.2.0", "strip-ansi": "7.1.2" diff --git a/packages/core/src/benchmark/markdown-link-benchmark.ts b/packages/core/src/benchmark/markdown-link-benchmark.ts new file mode 100644 index 000000000..0f2d36086 --- /dev/null +++ b/packages/core/src/benchmark/markdown-link-benchmark.ts @@ -0,0 +1,436 @@ +#!/usr/bin/env bun + +import { existsSync } from "node:fs" +import { mkdir, writeFile } from "node:fs/promises" +import { dirname, resolve } from "node:path" +import { pathToFileURL } from "node:url" +import { Command } from "commander" +import type { TextChunk } from "../text-buffer.js" +import type { SimpleHighlight } from "../lib/tree-sitter/types.js" + +type Implementation = "baseline" | "integrated" | "postpass" +type Stage = "conversion" | "native" + +interface Fixture { + content: string + highlights: SimpleHighlight[] +} + +interface Scenario { + name: string + description: string + fixtures: Fixture[] +} + +interface ScenarioResult { + name: string + description: string + finalChars: number + processedChars: number + highlightCount: number + chunkCount: number + linkedChunkCount: number + medianMs: number + p95Ms: number + minMs: number + maxMs: number + throughputMiBPerSecond: number +} + +interface Runtime { + treeSitterToTextChunks: ( + content: string, + highlights: SimpleHighlight[], + syntaxStyle: unknown, + options?: { enabled?: boolean; linkRanges?: Array<{ start: number; end: number; url: string }> }, + ) => TextChunk[] + detectMarkdownLinks?: ( + highlights: SimpleHighlight[], + context: { content: string; linkRanges?: Array<{ start: number; end: number; url: string }> }, + ) => SimpleHighlight[] + detectLinks?: (chunks: TextChunk[], context: { content: string; highlights: SimpleHighlight[] }) => TextChunk[] + syntaxStyle: { destroy(): void } + createStyledText: (chunks: TextChunk[]) => unknown + textBuffer?: { setStyledText(text: unknown): void; getPlainText(): string; destroy(): void } +} + +const SUITES = { + quick: { sizes: [16 * 1024], includeStress: false }, + default: { sizes: [1024, 16 * 1024], includeStress: true }, +} as const + +const program = new Command() +program + .name("markdown-link-benchmark") + .description("Benchmark Markdown link detection, chunk conversion, and optional native ingestion") + .option("-s, --suite ", "benchmark suite: quick, default", "default") + .option( + "--implementation ", + "implementation: baseline, integrated, postpass (historical source root required)", + "integrated", + ) + .option("--stage ", "stage: conversion, native", "conversion") + .option("--rounds ", "measured rounds", "7") + .option("--min-sample-ms ", "minimum duration per measured round", "200") + .option( + "--source-root ", + "packages/core root whose implementation is loaded", + resolve(import.meta.dir, "../.."), + ) + .option("--scenario ", "run one scenario") + .option("--list-scenarios", "list scenario names and exit") + .option("--json ", "write JSON results") + .parse(process.argv) + +const options = program.opts() +const suiteName = String(options.suite) as keyof typeof SUITES +const suite = SUITES[suiteName] +const implementation = String(options.implementation) as Implementation +const stage = String(options.stage) as Stage +const rounds = Math.max(1, Math.floor(Number(options.rounds))) +const minSampleMs = Math.max(1, Number(options.minSampleMs)) +const sourceRoot = resolve(String(options.sourceRoot)) +const scenarioFilter = options.scenario ? String(options.scenario) : undefined +const jsonPath = options.json ? resolve(String(options.json)) : undefined + +if (!suite) throw new Error(`Unknown suite: ${suiteName}`) +if (!(["baseline", "integrated", "postpass"] as const).includes(implementation)) { + throw new Error(`Unknown implementation: ${implementation}`) +} +if (!(["conversion", "native"] as const).includes(stage)) throw new Error(`Unknown stage: ${stage}`) +if (!Number.isFinite(rounds) || !Number.isFinite(minSampleMs)) throw new Error("Invalid timing options") + +const scenarios = createScenarios(suite) +if (options.listScenarios) { + for (const scenario of scenarios) console.log(scenario.name) + process.exit(0) +} + +const selectedScenarios = scenarioFilter ? scenarios.filter((scenario) => scenario.name === scenarioFilter) : scenarios +if (selectedScenarios.length === 0) throw new Error(`Unknown scenario: ${scenarioFilter}`) + +const runtime = await loadRuntime(sourceRoot, stage) +let sink = 0 +const results: ScenarioResult[] = [] + +try { + for (const scenario of selectedScenarios) { + const preflight = runScenarioOnce(runtime, scenario, implementation, stage, true) + const samples = measure(() => { + sink ^= runScenarioOnce(runtime, scenario, implementation, stage, false).checksum + }) + const sorted = samples.sort((left, right) => left - right) + const medianMs = percentile(sorted, 0.5) + const processedMiB = preflight.processedChars / (1024 * 1024) + + results.push({ + name: scenario.name, + description: scenario.description, + finalChars: scenario.fixtures.at(-1)?.content.length ?? 0, + processedChars: preflight.processedChars, + highlightCount: preflight.highlightCount, + chunkCount: preflight.chunkCount, + linkedChunkCount: preflight.linkedChunkCount, + medianMs, + p95Ms: percentile(sorted, 0.95), + minMs: sorted[0] ?? 0, + maxMs: sorted.at(-1) ?? 0, + throughputMiBPerSecond: medianMs > 0 ? processedMiB / (medianMs / 1000) : 0, + }) + } +} finally { + runtime.textBuffer?.destroy() + runtime.syntaxStyle.destroy() +} + +console.log( + `markdown link benchmark suite=${suiteName} implementation=${implementation} stage=${stage} rounds=${rounds} minSampleMs=${minSampleMs}`, +) +console.log(`sourceRoot=${sourceRoot}`) +console.table( + results.map((result) => ({ + scenario: result.name, + chars: result.finalChars, + processed: result.processedChars, + highlights: result.highlightCount, + chunks: result.chunkCount, + linked: result.linkedChunkCount, + medianMs: Number(result.medianMs.toFixed(4)), + p95Ms: Number(result.p95Ms.toFixed(4)), + MiBps: Number(result.throughputMiBPerSecond.toFixed(2)), + })), +) + +if (jsonPath) { + if (existsSync(jsonPath)) throw new Error(`Output file already exists: ${jsonPath}`) + await mkdir(dirname(jsonPath), { recursive: true }) + await writeFile( + jsonPath, + JSON.stringify( + { + timestamp: new Date().toISOString(), + suite: suiteName, + implementation, + stage, + rounds, + minSampleMs, + sourceRoot, + results, + }, + null, + 2, + ), + ) +} + +// Keep benchmark results observable without adding checksum work to the measured conversion. +if (sink === Number.MIN_SAFE_INTEGER) console.log(sink) + +function createScenarios(config: (typeof SUITES)[keyof typeof SUITES]): Scenario[] { + const result: Scenario[] = [] + for (const size of config.sizes) { + const suffix = formatSize(size) + result.push( + { + name: `plain_${suffix}`, + description: "URL-free Markdown", + fixtures: [buildPlainFixture(size)], + }, + { + name: `sparse_bare_${suffix}`, + description: "One bare URL per roughly 4 KiB", + fixtures: [buildBareFixture(size, 100)], + }, + { + name: `excluded_plain_${suffix}`, + description: "URL-free Markdown with many excluded code ranges", + fixtures: [buildExcludedFixture(size)], + }, + { + name: `dense_bare_${suffix}`, + description: "Bare URL on every line", + fixtures: [buildBareFixture(size, 1)], + }, + { + name: `dense_explicit_${suffix}`, + description: "Explicit Markdown link on every line", + fixtures: [buildExplicitFixture(size)], + }, + ) + } + + if (config.includeStress) { + result.push( + { + name: "plain_1m", + description: "URL-free 1 MiB stress case", + fixtures: [buildPlainFixture(1024 * 1024)], + }, + { + name: "sparse_bare_1m", + description: "Sparse bare URLs in a 1 MiB block", + fixtures: [buildBareFixture(1024 * 1024, 100)], + }, + { + name: "dense_bare_1m", + description: "Dense bare URLs in a 1 MiB block", + fixtures: [buildBareFixture(1024 * 1024, 1)], + }, + { + name: "dense_explicit_256k", + description: "Dense explicit links in a 256 KiB block", + fixtures: [buildExplicitFixture(256 * 1024)], + }, + { + name: "growing_sparse_bare_256k", + description: "Sixteen cumulative rescans at 16 KiB increments", + fixtures: Array.from({ length: 16 }, (_, index) => buildBareFixture((index + 1) * 16 * 1024, 100)), + }, + ) + } + return result +} + +function buildPlainFixture(size: number): Fixture { + const content = fit("ordinary markdown text without a link\n", size) + return { content, highlights: [[0, content.length, "spell"]] } +} + +function buildBareFixture(size: number, frequency: number): Fixture { + const plain = "ordinary markdown text without a link\n" + const linked = "ordinary markdown https://example.test/path?q=1 text\n" + const parts: string[] = [] + let line = 0 + let length = 0 + while (length < size) { + const next = line % frequency === 0 ? linked : plain + const remaining = size - length + const value = next.slice(0, remaining) + parts.push(value) + length += value.length + line++ + } + const content = parts.join("") + return { content, highlights: [[0, content.length, "spell"]] } +} + +function buildExcludedFixture(size: number): Fixture { + const line = "`inline code` ordinary markdown text\n" + const parts: string[] = [] + const highlights: SimpleHighlight[] = [] + let offset = 0 + while (offset + line.length <= size) { + parts.push(line) + highlights.push([offset, offset + 13, "markup.raw"]) + offset += line.length + } + if (offset < size) parts.push("x".repeat(size - offset)) + const content = parts.join("") + highlights.unshift([0, content.length, "spell"]) + return { content, highlights } +} + +function buildExplicitFixture(size: number): Fixture { + const parts: string[] = [] + const highlights: SimpleHighlight[] = [] + let offset = 0 + let index = 0 + + while (offset < size) { + const label = `label-${index}` + const url = `https://target-${index}.test/path` + const source = `[${label}](${url})\n` + if (offset + source.length > size) break + parts.push(source) + highlights.push( + [offset + 1, offset + 1 + label.length, "markup.link.label"], + [offset + label.length + 3, offset + label.length + 3 + url.length, "markup.link.url"], + ) + offset += source.length + index++ + } + + if (offset < size) parts.push("x".repeat(size - offset)) + const content = parts.join("") + highlights.unshift([0, content.length, "spell"]) + return { content, highlights } +} + +function fit(line: string, size: number): string { + return line.repeat(Math.ceil(size / line.length)).slice(0, size) +} + +function formatSize(size: number): string { + return size >= 1024 * 1024 ? `${size / (1024 * 1024)}m` : `${size / 1024}k` +} + +async function loadRuntime(root: string, selectedStage: Stage): Promise { + const importSource = (relativePath: string) => import(pathToFileURL(resolve(root, relativePath)).href) + const [{ treeSitterToTextChunks }, linkDetection, { SyntaxStyle }] = await Promise.all([ + importSource("src/lib/tree-sitter-styled-text.ts"), + importSource("src/lib/detect-links.ts"), + importSource("src/syntax-style.ts"), + ]) + const syntaxStyle = SyntaxStyle.fromStyles({ + default: {}, + spell: {}, + "markup.link.label": { underline: true }, + "markup.link.url": { underline: true }, + }) + + if (selectedStage === "conversion") { + return { + treeSitterToTextChunks, + detectMarkdownLinks: linkDetection.detectMarkdownLinks, + detectLinks: linkDetection.detectLinks, + syntaxStyle, + createStyledText: (chunks) => ({ chunks }), + } + } + + const [{ StyledText }, { TextBuffer }] = await Promise.all([ + importSource("src/lib/styled-text.ts"), + importSource("src/text-buffer.ts"), + ]) + return { + treeSitterToTextChunks, + detectMarkdownLinks: linkDetection.detectMarkdownLinks, + detectLinks: linkDetection.detectLinks, + syntaxStyle, + createStyledText: (chunks) => new StyledText(chunks), + textBuffer: TextBuffer.create("unicode"), + } +} + +function convert(runtime: Runtime, fixture: Fixture, selectedImplementation: Implementation): TextChunk[] { + const context: { content: string; linkRanges?: Array<{ start: number; end: number; url: string }> } = { + content: fixture.content, + } + const highlights = fixture.highlights + if (selectedImplementation === "integrated") runtime.detectMarkdownLinks!(highlights, context) + const chunks = runtime.treeSitterToTextChunks(fixture.content, highlights, runtime.syntaxStyle, { + enabled: false, + linkRanges: context.linkRanges, + }) + if (selectedImplementation === "postpass") { + if (!runtime.detectLinks) throw new Error("The selected source root does not provide the post-pass detector") + runtime.detectLinks(chunks, { content: fixture.content, highlights: fixture.highlights }) + } + return chunks +} + +function runScenarioOnce( + runtime: Runtime, + scenario: Scenario, + selectedImplementation: Implementation, + selectedStage: Stage, + inspect: boolean, +): { checksum: number; processedChars: number; highlightCount: number; chunkCount: number; linkedChunkCount: number } { + let checksum = 0 + let processedChars = 0 + let highlightCount = 0 + let chunkCount = 0 + let linkedChunkCount = 0 + + for (const fixture of scenario.fixtures) { + const chunks = convert(runtime, fixture, selectedImplementation) + if (selectedStage === "native") runtime.textBuffer!.setStyledText(runtime.createStyledText(chunks)) + checksum = (checksum * 33 + chunks.length) | 0 + + if (!inspect) continue + const rendered = chunks.map((chunk) => chunk.text).join("") + if (rendered !== fixture.content) throw new Error(`${scenario.name}: conversion changed rendered text`) + if (selectedStage === "native" && runtime.textBuffer!.getPlainText() !== fixture.content) { + throw new Error(`${scenario.name}: native ingestion changed rendered text`) + } + processedChars += fixture.content.length + highlightCount += fixture.highlights.length + chunkCount += chunks.length + linkedChunkCount += chunks.reduce((count, chunk) => count + (chunk.link ? 1 : 0), 0) + } + + return { checksum, processedChars, highlightCount, chunkCount, linkedChunkCount } +} + +function measure(operation: () => void): number[] { + for (let index = 0; index < 3; index++) operation() + const samples: number[] = [] + + for (let round = 0; round < rounds; round++) { + let iterations = 1 + let elapsed = 0 + do { + const start = performance.now() + for (let index = 0; index < iterations; index++) operation() + elapsed = performance.now() - start + if (elapsed < minSampleMs) + iterations = Math.max(iterations + 1, Math.ceil((iterations * minSampleMs) / Math.max(elapsed, 0.01))) + } while (elapsed < minSampleMs) + samples.push(elapsed / iterations) + } + return samples +} + +function percentile(sorted: number[], value: number): number { + return sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil(sorted.length * value) - 1))] ?? 0 +} diff --git a/packages/core/src/lib/detect-links.test.ts b/packages/core/src/lib/detect-links.test.ts index b53eafc72..d6cf974c5 100644 --- a/packages/core/src/lib/detect-links.test.ts +++ b/packages/core/src/lib/detect-links.test.ts @@ -1,98 +1,138 @@ -import { test, expect, describe } from "bun:test" -import { detectLinks } from "./detect-links.js" -import type { TextChunk } from "../text-buffer.js" +import { describe, expect, test } from "bun:test" +import { admitLinkTarget, detectBareLinks, detectMarkdownLinks, detectSourceLinks } from "./detect-links.js" import type { SimpleHighlight } from "./tree-sitter/types.js" -import { RGBA } from "./RGBA.js" -function chunk(text: string): TextChunk { - return { __isChunk: true, text, fg: RGBA.fromInts(255, 255, 255, 255), attributes: 0 } -} - -describe("detectLinks", () => { - test("should set link on markup.link.url chunks", () => { - const content = "[Click here](https://example.com)" +describe("detectSourceLinks", () => { + test("uses Marked-resolved explicit destinations and keeps labels continuous", () => { + const content = + "[a&b](https://example.test/?a=1&b=2) [angle]() [escape](https://x.test/a\\(b\\))" const highlights: SimpleHighlight[] = [ - [0, 1, "markup.link"], - [1, 11, "markup.link.label"], - [11, 13, "markup.link"], - [13, 32, "markup.link.url"], - [32, 33, "markup.link"], + [1, 8, "markup.link.label"], + [10, 43, "markup.link.url"], + [46, 51, "markup.link.label"], + [53, 75, "markup.link.url"], + [78, 84, "markup.link.label"], + [86, 107, "markup.link.url"], ] - const chunks = [chunk("["), chunk("Click here"), chunk("]("), chunk("https://example.com"), chunk(")")] - - const result = detectLinks(chunks, { content, highlights }) - expect(result.find((c) => c.text === "https://example.com")!.link).toEqual({ url: "https://example.com" }) - expect(result.find((c) => c.text === "Click here")!.link).toEqual({ url: "https://example.com" }) + expect( + detectSourceLinks(content, highlights).map(({ start, end, url }) => [content.slice(start, end), url]), + ).toEqual([ + ["a&b", "https://example.test/?a=1&b=2"], + ["https://example.test/?a=1&b=2", "https://example.test/?a=1&b=2"], + ["angle", "https://x.test/a%20b"], + ["", "https://x.test/a%20b"], + ["escape", "https://x.test/a(b)"], + ["https://x.test/a\\(b\\)", "https://x.test/a(b)"], + ]) }) - test("should set link on string.special.url chunks", () => { - const content = "// see https://example.com for details" - const highlights: SimpleHighlight[] = [ - [0, 38, "comment"], - [7, 26, "string.special.url"], - ] - const chunks = [chunk("// see "), chunk("https://example.com"), chunk(" for details")] + test("keeps explicit links authoritative without a dense quadratic fixture", () => { + const content = Array.from( + { length: 64 }, + (_, index) => `[https://label${index}.test](https://target${index}.test)`, + ).join(" ") + const highlights: SimpleHighlight[] = [] + for (const match of content.matchAll(/\[([^\]]+)\]\(([^)]+)\)/gu)) { + const label = match.index + 1 + const url = label + match[1].length + 2 + highlights.push( + [label, label + match[1].length, "markup.link.label"], + [url, url + match[2].length, "markup.link.url"], + ) + } - const result = detectLinks(chunks, { content, highlights }) + const links = detectSourceLinks(content, highlights) + expect(links).toHaveLength(128) + expect(links.every((link) => !link.url.includes("label"))).toBe(true) + }) + + test("preserves escaped and semicolonless ampersands in explicit targets", () => { + const content = + "[escaped](https://x.test/?q=\\©) [literal](https://x.test/?a=1©) [entity](https://x.test/?a=1&b=2)" + const highlights: SimpleHighlight[] = [] + for (const match of content.matchAll(/\[([^\]]+)\]\(([^)]+)\)/gu)) { + const labelStart = match.index + 1 + const urlStart = labelStart + match[1].length + 2 + highlights.push( + [labelStart, labelStart + match[1].length, "markup.link.label"], + [urlStart, urlStart + match[2].length, "markup.link.url"], + ) + } - expect(result.find((c) => c.text === "https://example.com")!.link).toEqual({ url: "https://example.com" }) + expect(detectSourceLinks(content, highlights).map((link) => link.url)).toEqual([ + "https://x.test/?q=©", + "https://x.test/?q=©", + "https://x.test/?a=1©", + "https://x.test/?a=1©", + "https://x.test/?a=1&b=2", + "https://x.test/?a=1&b=2", + ]) }) - test("should not set link on non-URL chunks", () => { - const content = "const x = 42" + test("links a normal label after an escaped image marker", () => { + const content = "\\![label](https://x.test)" const highlights: SimpleHighlight[] = [ - [0, 5, "keyword"], - [6, 7, "variable"], - [10, 12, "number"], + [3, 8, "markup.link.label"], + [10, 24, "markup.link.url"], ] - const chunks = [chunk("const"), chunk(" "), chunk("x"), chunk(" = "), chunk("42")] - const result = detectLinks(chunks, { content, highlights }) + expect(detectSourceLinks(content, highlights)).toEqual([ + { start: 3, end: 8, url: "https://x.test" }, + { start: 10, end: 24, url: "https://x.test" }, + ]) + }) - for (const c of result) { - expect(c.link).toBeUndefined() - } + test("decorates source ranges through the highlight hook", () => { + const content = "plain https://x.test text" + const highlights: SimpleHighlight[] = [[0, content.length, "spell"]] + const context: { content: string; linkRanges?: Array<{ start: number; end: number; url: string }> } = { content } + + expect(detectMarkdownLinks(highlights, context)).toBe(highlights) + expect(context.linkRanges).toEqual([{ start: 6, end: 20, url: "https://x.test" }]) }) +}) - test("should return chunks unchanged when no URL scopes exist", () => { - const content = "hello world" - const highlights: SimpleHighlight[] = [[0, 5, "keyword"]] - const chunks = [chunk("hello"), chunk(" world")] +describe("parser-owned bare URLs", () => { + test("uses excluded ranges as hard boundaries and resumes after them", () => { + const content = "https://safe.example`https://code.example`HTTPS://AFTER.EXAMPLE" + const codeStart = content.indexOf("`") + expect(detectBareLinks(content, [{ start: codeStart, end: content.lastIndexOf("`") + 1 }])).toEqual([ + { start: 0, end: 20, url: "https://safe.example" }, + { start: 42, end: 63, url: "HTTPS://AFTER.EXAMPLE" }, + ]) + }) - const result = detectLinks(chunks, { content, highlights }) + test("matches Marked's case and punctuation semantics", () => { + expect( + detectBareLinks("HTTPS://EXAMPLE.COM, https://x.test/a(b). https://x.test/foo).").map((link) => link.url), + ).toEqual(["HTTPS://EXAMPLE.COM", "https://x.test/a(b)", "https://x.test/foo"]) + }) - expect(result).toBe(chunks) + test("preserves semicolonless entity names in bare URL targets", () => { + expect(detectBareLinks("https://x.test/?a=1© https://x.test/?a=1¬").map((link) => link.url)).toEqual([ + "https://x.test/?a=1©", + "https://x.test/?a=1¬", + ]) }) - test("should detect links when chunks have concealed text", () => { - // Original content: [Click here](https://example.com) - // With concealment, `[` and `]` are concealed to empty strings, - // and `(` and `)` are concealed to empty strings. - // This means chunk text lengths don't match original byte offsets. - const content = "[Click here](https://example.com)" - const highlights: SimpleHighlight[] = [ - [0, 1, "markup.link"], // [ - [1, 11, "markup.link.label"], // Click here - [11, 13, "markup.link"], // ]( - [13, 32, "markup.link.url"], // https://example.com - [32, 33, "markup.link"], // ) - ] - // Simulate concealed chunks: `[` -> "", `](` -> " ", `)` -> "" - // The URL and label chunks remain unchanged. - const chunks = [ - chunk(""), // concealed `[` - chunk("Click here"), // label, unchanged - chunk(" "), // concealed `](` - chunk("https://example.com"), // URL, unchanged - chunk(""), // concealed `)` - ] + test("handles the former quadratic alternating suffix at review scale", () => { + const content = `https://x.test/${")] }".replace(" ", "").repeat(16_000)}` + expect(detectBareLinks(content)).toHaveLength(1) + }) - const result = detectLinks(chunks, { content, highlights }) + test("rejects empty and decoded C0, DEL, and C1 targets", () => { + expect(admitLinkTarget("")).toBeUndefined() + for (const control of ["\0", "\x07", "\x1b", "\x7f", "\x80", "\x9c"]) { + expect(admitLinkTarget(`https://safe.example/${control}`)).toBeUndefined() + } - // The URL chunk should still get its link despite concealed offsets - expect(result.find((c) => c.text === "https://example.com")!.link).toEqual({ url: "https://example.com" }) - // The label chunk should also get the link - expect(result.find((c) => c.text === "Click here")!.link).toEqual({ url: "https://example.com" }) + const content = "[x](https://safe.example/)" + expect( + detectSourceLinks(content, [ + [1, 2, "markup.link.label"], + [4, 29, "markup.link.url"], + ]), + ).toEqual([]) }) }) diff --git a/packages/core/src/lib/detect-links.ts b/packages/core/src/lib/detect-links.ts index b7fa14e08..6f87356c4 100644 --- a/packages/core/src/lib/detect-links.ts +++ b/packages/core/src/lib/detect-links.ts @@ -1,56 +1,140 @@ -import type { TextChunk } from "../text-buffer.js" -import type { SimpleHighlight } from "./tree-sitter/types.js" +import { decodeHTMLStrict } from "entities" +import { Lexer, type Token, type Tokens } from "marked" +import type { LinkRange, SimpleHighlight } from "./tree-sitter/types.js" -const URL_SCOPES = ["markup.link.url", "string.special.url"] +export type SourceLink = LinkRange -export function detectLinks( - chunks: TextChunk[], - context: { content: string; highlights: SimpleHighlight[] }, -): TextChunk[] { - const content = context.content - const highlights = context.highlights +const URL_SCOPES = new Set(["markup.link.url", "string.special.url"]) +const EXCLUDED_SCOPES = new Set(["markup.link.label", "markup.raw", "markup.raw.inline", "markup.raw.block"]) +const CONTROL_CHARACTERS = /[\u0000-\u001f\u007f-\u009f]/u +const URL_START = /https?:\/\//giu +const MARKDOWN_ENTITY = /&(?:#\d+|#[xX][\dA-Fa-f]+|[A-Za-z][A-Za-z\d]+);/gu - const ranges: Array<{ start: number; end: number; url: string }> = [] +export function admitLinkTarget(target: string): string | undefined { + if (!target || CONTROL_CHARACTERS.test(target)) return undefined + return target +} - for (let i = 0; i < highlights.length; i++) { - const [start, end, group] = highlights[i] - if (!URL_SCOPES.includes(group)) continue +export function detectMarkdownLinks( + highlights: SimpleHighlight[], + context: { content: string; linkRanges?: LinkRange[] }, +): SimpleHighlight[] { + const links = detectSourceLinks(context.content, highlights) + if (links.length > 0) context.linkRanges = links + return highlights +} - const url = content.slice(start, end) - ranges.push({ start, end, url }) +export function detectSourceLinks(content: string, highlights: SimpleHighlight[]): SourceLink[] { + const detected = detectExplicitLinks(content, highlights) + const explicit = detected.links.sort((left, right) => left.start - right.start || left.end - right.end) + const bare = detectBareLinks(content, detected.excluded) + const links: SourceLink[] = [] + let explicitIndex = 0 - for (let j = i - 1; j >= 0; j--) { - const [labelStart, labelEnd, prev] = highlights[j] - if (prev === "markup.link.label") { - ranges.push({ start: labelStart, end: labelEnd, url }) - break - } - if (!prev.startsWith("markup.link")) break + for (const candidate of bare) { + while (explicit[explicitIndex] && explicit[explicitIndex].end <= candidate.start) { + links.push(explicit[explicitIndex++]) } + if (explicit[explicitIndex] && explicit[explicitIndex].start < candidate.end) continue + links.push(candidate) } + while (explicit[explicitIndex]) links.push(explicit[explicitIndex++]) + return links +} + +export function detectBareLinks(text: string, excluded: Array<{ start: number; end: number }> = []): SourceLink[] { + const links: SourceLink[] = [] + const ranges = [...excluded].sort((left, right) => left.start - right.start) + let excludedIndex = 0 + let match: RegExpExecArray | null + URL_START.lastIndex = 0 - if (ranges.length === 0) return chunks + while ((match = URL_START.exec(text))) { + const start = match.index + while (ranges[excludedIndex] && ranges[excludedIndex].end <= start) excludedIndex++ + if (ranges[excludedIndex] && start >= ranges[excludedIndex].start) { + URL_START.lastIndex = ranges[excludedIndex].end + continue + } - // Use content.indexOf to find each chunk's position in the original content. - // This handles concealed text correctly because concealed chunks are either - // empty (length 0, skipped) or single-char replacements (length 1, skipped). - // Non-concealed chunks with length > 1 are exact substrings of content in order. - let contentPos = 0 - for (const chunk of chunks) { - if (chunk.text.length <= 1) continue + const lowerBound = ranges[excludedIndex - 1]?.end ?? 0 + const upperBound = ranges[excludedIndex]?.start ?? text.length + let segmentStart = start + let segmentEnd = start + while (segmentStart > lowerBound && !/\s/u.test(text[segmentStart - 1])) segmentStart-- + while (segmentEnd < upperBound && !/\s/u.test(text[segmentEnd])) segmentEnd++ + const source = text.slice(segmentStart, segmentEnd) + collectBareTokens(Lexer.lexInline(source), source, segmentStart, links) + URL_START.lastIndex = segmentEnd + } + return links +} - const idx = content.indexOf(chunk.text, contentPos) - if (idx < 0) continue +function collectBareTokens(tokens: Token[], source: string, sourceStart: number, links: SourceLink[]): void { + let offset = 0 + for (const token of tokens) { + const start = source.indexOf(token.raw, offset) + if (start < 0) continue + if (token.type === "link" && /^https?:\/\//iu.test(token.raw)) { + const url = resolveMarkedLinkTarget(token as Tokens.Link) + if (url) links.push({ start: sourceStart + start, end: sourceStart + start + token.raw.length, url }) + } else if ("tokens" in token && Array.isArray(token.tokens)) { + collectBareTokens(token.tokens, token.raw, sourceStart + start, links) + } + offset = start + token.raw.length + } +} - for (const range of ranges) { - if (idx < range.end && idx + chunk.text.length > range.start) { - chunk.link = { url: range.url } +function detectExplicitLinks(content: string, highlights: SimpleHighlight[]) { + const links: SourceLink[] = [] + const excluded: Array<{ start: number; end: number }> = [] + + for (let index = 0; index < highlights.length; index++) { + const [start, end, group] = highlights[index] + if (EXCLUDED_SCOPES.has(group) || URL_SCOPES.has(group)) excluded.push({ start, end }) + if (!URL_SCOPES.has(group)) continue + + const url = resolveMarkdownDestination(content.slice(start, end)) + if (!url) continue + links.push({ start, end, url }) + if (group !== "markup.link.url") continue + + for (let previous = index - 1; previous >= 0; previous--) { + const [labelStart, labelEnd, previousGroup] = highlights[previous] + if (previousGroup === "markup.link.label" && /^\]\(\s* + isEscaped(source, offset) ? entity : decodeHTMLStrict(entity), + ) +} - return chunks +function isEscaped(source: string, offset: number): boolean { + let backslashes = 0 + for (let index = offset - 1; index >= 0 && source[index] === "\\"; index--) backslashes++ + return backslashes % 2 === 1 } diff --git a/packages/core/src/lib/index.ts b/packages/core/src/lib/index.ts index ad0cd33fd..2c24dbb5e 100644 --- a/packages/core/src/lib/index.ts +++ b/packages/core/src/lib/index.ts @@ -19,4 +19,3 @@ export * from "./extmarks.js" export * from "./terminal-palette.js" export * from "./paste.js" export * from "./clipboard.js" -export { detectLinks } from "./detect-links.js" diff --git a/packages/core/src/lib/tree-sitter-styled-text.test.ts b/packages/core/src/lib/tree-sitter-styled-text.test.ts index e84b7e810..302f07268 100644 --- a/packages/core/src/lib/tree-sitter-styled-text.test.ts +++ b/packages/core/src/lib/tree-sitter-styled-text.test.ts @@ -1043,6 +1043,28 @@ Normal paragraph with [link](https://example.com).` expect(chunk.attributes).toBe(expectedAttributes) }) + test("applies adjacent link metadata without changing text", () => { + const content = "beforefirstsecondafter" + const testStyle = SyntaxStyle.fromStyles({ default: {} }) + const highlights: SimpleHighlight[] = [] + + const chunks = treeSitterToTextChunks(content, highlights, testStyle, { + linkRanges: [ + { start: 6, end: 11, url: "https://first.test" }, + { start: 11, end: 17, url: "https://second.test" }, + ], + }) + testStyle.destroy() + + expect(chunks.map((chunk) => chunk.text).join("")).toBe(content) + expect(chunks.map(({ text, link }) => [text, link?.url])).toEqual([ + ["before", undefined], + ["first", "https://first.test"], + ["second", "https://second.test"], + ["after", undefined], + ]) + }) + test("should handle style inheritance when parent only sets attributes", () => { const mockHighlights: SimpleHighlight[] = [ [0, 15, "container"], // Parent: only underline diff --git a/packages/core/src/lib/tree-sitter-styled-text.ts b/packages/core/src/lib/tree-sitter-styled-text.ts index 9b5731812..9578859bb 100644 --- a/packages/core/src/lib/tree-sitter-styled-text.ts +++ b/packages/core/src/lib/tree-sitter-styled-text.ts @@ -2,7 +2,7 @@ import type { TextChunk } from "../text-buffer.js" import { StyledText } from "./styled-text.js" import { SyntaxStyle, type StyleDefinition } from "../syntax-style.js" import { TreeSitterClient } from "./tree-sitter/client.js" -import type { SimpleHighlight } from "./tree-sitter/types.js" +import type { LinkRange, SimpleHighlight } from "./tree-sitter/types.js" import { createTextAttributes } from "../utils.js" import { registerEnvVar, env } from "./env.js" @@ -11,6 +11,7 @@ registerEnvVar({ name: "OTUI_TS_STYLE_WARN", default: false, description: "Enabl interface TextChunkOptions { enabled?: boolean baseHighlight?: string + linkRanges?: LinkRange[] } interface Boundary { @@ -36,6 +37,41 @@ function shouldSuppressInInjection(group: string, meta: any): boolean { return group === "markup.raw.block" } +function pushLinkedChunks( + chunks: TextChunk[], + content: string, + links: LinkRange[], + linkIndex: number, + start: number, + end: number, + chunk: TextChunk, + replacement?: string, +): number { + while (links[linkIndex] && links[linkIndex].end <= start) linkIndex++ + if (replacement !== undefined) { + const link = links[linkIndex] + chunks.push({ ...chunk, text: replacement, link: link && link.start < end ? { url: link.url } : undefined }) + return linkIndex + } + if (!links[linkIndex] || links[linkIndex].start >= end) { + chunks.push({ ...chunk, text: content.slice(start, end) }) + return linkIndex + } + + let offset = start + for (let index = linkIndex; links[index] && links[index].start < end; index++) { + const link = links[index] + const linkStart = Math.max(offset, link.start) + const linkEnd = Math.min(end, link.end) + if (linkEnd <= linkStart) continue + if (offset < linkStart) chunks.push({ ...chunk, text: content.slice(offset, linkStart) }) + chunks.push({ ...chunk, text: content.slice(linkStart, linkEnd), link: { url: link.url } }) + offset = linkEnd + } + if (offset < end) chunks.push({ ...chunk, text: content.slice(offset, end) }) + return linkIndex +} + export function treeSitterToTextChunks( content: string, highlights: SimpleHighlight[], @@ -49,6 +85,7 @@ export function treeSitterToTextChunks( const injectionContainerRanges: Array<{ start: number; end: number }> = [] const boundaries: Boundary[] = [] + const links = options?.linkRanges ?? [] for (let i = 0; i < highlights.length; i++) { const [start, end, , meta] = highlights[i] @@ -71,13 +108,12 @@ export function treeSitterToTextChunks( const activeHighlights = new Set() let currentOffset = 0 + let linkIndex = 0 for (let i = 0; i < boundaries.length; i++) { const boundary = boundaries[i] if (currentOffset < boundary.offset && activeHighlights.size > 0) { - const segmentText = content.slice(currentOffset, boundary.offset) - const activeGroups: Array<{ group: string; meta: any; index: number }> = [] for (const idx of activeHighlights) { const [, , group, meta] = highlights[idx] @@ -104,8 +140,8 @@ export function treeSitterToTextChunks( } if (replacementText) { - chunks.push({ - __isChunk: true, + const chunk = { + __isChunk: true as const, text: replacementText, fg: defaultStyle?.fg, bg: defaultStyle?.bg, @@ -117,7 +153,21 @@ export function treeSitterToTextChunks( dim: defaultStyle.dim, }) : 0, - }) + } + if (links.length > 0) { + linkIndex = pushLinkedChunks( + chunks, + content, + links, + linkIndex, + currentOffset, + boundary.offset, + chunk, + replacementText, + ) + } else { + chunks.push(chunk) + } } } else { const insideInjectionContainer = injectionContainerRanges.some( @@ -184,9 +234,9 @@ export function treeSitterToTextChunks( // Use merged style, falling back to default if nothing was merged const finalStyle = Object.keys(mergedStyle).length > 0 ? mergedStyle : defaultStyle - chunks.push({ - __isChunk: true, - text: segmentText, + const chunk = { + __isChunk: true as const, + text: content.slice(currentOffset, boundary.offset), fg: finalStyle?.fg, bg: finalStyle?.bg, attributes: finalStyle @@ -197,14 +247,18 @@ export function treeSitterToTextChunks( dim: finalStyle.dim, }) : 0, - }) + } + if (links.length > 0) { + linkIndex = pushLinkedChunks(chunks, content, links, linkIndex, currentOffset, boundary.offset, chunk) + } else { + chunks.push(chunk) + } } } else if (currentOffset < boundary.offset) { - const text = content.slice(currentOffset, boundary.offset) const style = baseStyle ?? defaultStyle - chunks.push({ - __isChunk: true, - text, + const chunk = { + __isChunk: true as const, + text: content.slice(currentOffset, boundary.offset), fg: style?.fg, bg: style?.bg, attributes: style @@ -215,7 +269,12 @@ export function treeSitterToTextChunks( dim: style.dim, }) : 0, - }) + } + if (links.length > 0) { + linkIndex = pushLinkedChunks(chunks, content, links, linkIndex, currentOffset, boundary.offset, chunk) + } else { + chunks.push(chunk) + } } if (boundary.type === "start") { @@ -258,11 +317,10 @@ export function treeSitterToTextChunks( } if (currentOffset < content.length) { - const text = content.slice(currentOffset) const style = baseStyle ?? defaultStyle - chunks.push({ - __isChunk: true, - text, + const chunk = { + __isChunk: true as const, + text: content.slice(currentOffset), fg: style?.fg, bg: style?.bg, attributes: style @@ -273,7 +331,12 @@ export function treeSitterToTextChunks( dim: style.dim, }) : 0, - }) + } + if (links.length > 0) { + pushLinkedChunks(chunks, content, links, linkIndex, currentOffset, content.length, chunk) + } else { + chunks.push(chunk) + } } return chunks diff --git a/packages/core/src/lib/tree-sitter/assets/markdown/highlights.scm b/packages/core/src/lib/tree-sitter/assets/markdown/highlights.scm index 5eb9f6a6e..037c198a3 100644 --- a/packages/core/src/lib/tree-sitter/assets/markdown/highlights.scm +++ b/packages/core/src/lib/tree-sitter/assets/markdown/highlights.scm @@ -87,6 +87,8 @@ (link_destination) @markup.link.url +(html_block) @markup.raw.block + [ (link_title) (link_label) diff --git a/packages/core/src/lib/tree-sitter/assets/markdown_inline/highlights.scm b/packages/core/src/lib/tree-sitter/assets/markdown_inline/highlights.scm index 7fcaac203..dbb6f525c 100644 --- a/packages/core/src/lib/tree-sitter/assets/markdown_inline/highlights.scm +++ b/packages/core/src/lib/tree-sitter/assets/markdown_inline/highlights.scm @@ -89,6 +89,8 @@ (image_description) ] @markup.link.label +(html_tag) @markup.raw.inline + ; Replace common HTML entities. ((entity_reference) @character.special (#eq? @character.special " ") diff --git a/packages/core/src/lib/tree-sitter/types.ts b/packages/core/src/lib/tree-sitter/types.ts index b9848ba59..b3612dfe3 100644 --- a/packages/core/src/lib/tree-sitter/types.ts +++ b/packages/core/src/lib/tree-sitter/types.ts @@ -20,6 +20,12 @@ export interface HighlightMeta { export type SimpleHighlight = [number, number, string, HighlightMeta?] +export interface LinkRange { + start: number + end: number + url: string +} + export interface InjectionMapping { // Maps tree-sitter node types to target filetypes nodeTypes?: { [nodeType: string]: string } diff --git a/packages/core/src/renderables/Code.test.ts b/packages/core/src/renderables/Code.test.ts index 4743d4b91..d5d08eedd 100644 --- a/packages/core/src/renderables/Code.test.ts +++ b/packages/core/src/renderables/Code.test.ts @@ -1229,6 +1229,39 @@ test("CodeRenderable - onChunks callback can transform chunks when highlights ar expect(codeRenderable.plainText).toBe("HELLO") }) +test("CodeRenderable - onHighlight can add link ranges when highlights are empty", async () => { + const syntaxStyle = SyntaxStyle.fromStyles({ default: {} }) + const mockClient = new MockTreeSitterClient() + mockClient.setMockResult({ highlights: [] }) + let chunks: Array<{ text: string; link?: { url: string } }> = [] + + const codeRenderable = new CodeRenderable(currentRenderer, { + id: "test-code-link-ranges", + content: "visit example", + filetype: "plaintext", + syntaxStyle, + treeSitterClient: mockClient, + onHighlight: (highlights, context) => { + context.linkRanges = [{ start: 6, end: 13, url: "https://example.test" }] + return highlights + }, + }) + const textBuffer = (codeRenderable as any).textBuffer + const setStyledText = textBuffer.setStyledText.bind(textBuffer) + textBuffer.setStyledText = (styledText: { chunks: typeof chunks }) => { + chunks = styledText.chunks + setStyledText(styledText) + } + + currentRenderer.root.add(codeRenderable) + await resolveMockHighlights(codeRenderable, mockClient) + + expect(chunks.map(({ text, link }) => [text, link?.url])).toEqual([ + ["visit ", undefined], + ["example", "https://example.test"], + ]) +}) + test("CodeRenderable - baseHighlight applies a style when parser highlights are empty", async () => { const quoteColor = RGBA.fromValues(0.25, 0.5, 0.75, 1) const syntaxStyle = SyntaxStyle.fromStyles({ diff --git a/packages/core/src/renderables/Code.ts b/packages/core/src/renderables/Code.ts index 7b4819394..df59b3cb4 100644 --- a/packages/core/src/renderables/Code.ts +++ b/packages/core/src/renderables/Code.ts @@ -4,7 +4,7 @@ import { SyntaxStyle } from "../syntax-style.js" import { getTreeSitterClient, TreeSitterClient } from "../lib/tree-sitter/index.js" import { TextBufferRenderable, type TextBufferOptions } from "./TextBufferRenderable.js" import type { OptimizedBuffer } from "../buffer.js" -import type { SimpleHighlight } from "../lib/tree-sitter/types.js" +import type { LinkRange, SimpleHighlight } from "../lib/tree-sitter/types.js" import type { TextChunk } from "../text-buffer.js" import { treeSitterToTextChunks } from "../lib/tree-sitter-styled-text.js" @@ -12,6 +12,7 @@ export interface HighlightContext { content: string filetype: string syntaxStyle: SyntaxStyle + linkRanges?: LinkRange[] } export type OnHighlightCallback = ( @@ -339,6 +340,7 @@ export class CodeRenderable extends TextBufferRenderable { if (this.isDestroyed) return let highlights = result.highlights ?? [] + let linkRanges: LinkRange[] | undefined if (this._onHighlight && highlights.length >= 0) { const context: HighlightContext = { @@ -350,6 +352,7 @@ export class CodeRenderable extends TextBufferRenderable { if (modified !== undefined) { highlights = modified } + linkRanges = context.linkRanges } if (snapshotId !== this._highlightSnapshotId) { @@ -365,17 +368,19 @@ export class CodeRenderable extends TextBufferRenderable { } } - if (highlights.length > 0 || this._onChunks || this._baseHighlight) { + if (highlights.length > 0 || linkRanges?.length || this._onChunks || this._baseHighlight) { const context: ChunkRenderContext = { content, filetype, syntaxStyle: this._syntaxStyle, highlights, + linkRanges, } let chunks = treeSitterToTextChunks(content, highlights, this._syntaxStyle, { enabled: this._conceal, baseHighlight: this._baseHighlight, + linkRanges, }) // onChunks may rewrite text arbitrarily, so the conceal-only source map would be invalid. const renderedLineSources = this._onChunks ? undefined : this.getConcealLinesSourceMap(content, highlights) diff --git a/packages/core/src/renderables/Markdown.ts b/packages/core/src/renderables/Markdown.ts index 3de7e5ce5..c3bffe5b1 100644 --- a/packages/core/src/renderables/Markdown.ts +++ b/packages/core/src/renderables/Markdown.ts @@ -6,7 +6,7 @@ import { createTextAttributes } from "../utils.js" import type { BorderStyle } from "../lib/border.js" import { RGBA, parseColor, type ColorInput } from "../lib/RGBA.js" import { Lexer, type MarkedToken, type Token, type Tokens } from "marked" -import { CodeRenderable, type OnChunksCallback } from "./Code.js" +import { CodeRenderable } from "./Code.js" import { BoxRenderable } from "./Box.js" import { StyledText } from "../lib/styled-text.js" import { TextRenderable } from "./Text.js" @@ -21,7 +21,7 @@ import type { TreeSitterClient } from "../lib/tree-sitter/index.js" import { infoStringToFiletype } from "../lib/tree-sitter/resolve-ft.js" import { parseMarkdownIncremental, type ParseState } from "./markdown-parser.js" import type { OptimizedBuffer } from "../buffer.js" -import { detectLinks } from "../lib/detect-links.js" +import { detectMarkdownLinks, resolveMarkedLinkTarget } from "../lib/detect-links.js" export type MarkdownTableStyle = "grid" | "columns" @@ -275,12 +275,6 @@ export class MarkdownRenderable extends Renderable { _blockStates: BlockState[] = [] _stableBlockCount = 0 private _styleDirty: boolean = false - private _linkifyMarkdownChunks: OnChunksCallback = (chunks, context) => - detectLinks(chunks, { - content: context.content, - highlights: context.highlights, - }) - protected _contentDefaultOptions = { content: "", conceal: true, @@ -543,13 +537,18 @@ export class MarkdownRenderable extends Renderable { break case "link": { - const linkHref = { url: token.href } + const target = resolveMarkedLinkTarget(token) + if (!target) { + chunks.push(this.createDefaultChunk(token.text.replace(/[\u0000-\u001f\u007f-\u009f]/gu, ""))) + break + } + const linkHref = { url: target } if (this._conceal) { for (const child of token.tokens) { this.renderInlineTokenWithStyle(child as MarkedToken, chunks, "markup.link.label", linkHref) } chunks.push(this.createChunk(" (", "markup.link", linkHref)) - chunks.push(this.createChunk(token.href, "markup.link.url", linkHref)) + chunks.push(this.createChunk(target, "markup.link.url", linkHref)) chunks.push(this.createChunk(")", "markup.link", linkHref)) } else { chunks.push(this.createChunk("[", "markup.link", linkHref)) @@ -557,21 +556,26 @@ export class MarkdownRenderable extends Renderable { this.renderInlineTokenWithStyle(child as MarkedToken, chunks, "markup.link.label", linkHref) } chunks.push(this.createChunk("](", "markup.link", linkHref)) - chunks.push(this.createChunk(token.href, "markup.link.url", linkHref)) + chunks.push(this.createChunk(target, "markup.link.url", linkHref)) chunks.push(this.createChunk(")", "markup.link", linkHref)) } break } case "image": { - const imageHref = { url: token.href } + const target = resolveMarkedLinkTarget(token) + if (!target) { + chunks.push(this.createDefaultChunk((token.text || "image").replace(/[\u0000-\u001f\u007f-\u009f]/gu, ""))) + break + } + const imageHref = { url: target } if (this._conceal) { chunks.push(this.createChunk(token.text || "image", "markup.link.label", imageHref)) } else { chunks.push(this.createChunk("![", "markup.link", imageHref)) chunks.push(this.createChunk(token.text || "", "markup.link.label", imageHref)) chunks.push(this.createChunk("](", "markup.link", imageHref)) - chunks.push(this.createChunk(token.href, "markup.link.url", imageHref)) + chunks.push(this.createChunk(target, "markup.link.url", imageHref)) chunks.push(this.createChunk(")", "markup.link", imageHref)) } break @@ -631,7 +635,6 @@ export class MarkdownRenderable extends Renderable { content: string, id: string, marginBottom: number = 0, - onChunks: OnChunksCallback = this._linkifyMarkdownChunks, baseHighlight?: string, initialStyledText?: StyledText, ): CodeRenderable { @@ -647,7 +650,7 @@ export class MarkdownRenderable extends Renderable { streaming: true, initialStyledText, baseHighlight, - onChunks, + onHighlight: detectMarkdownLinks, treeSitterClient: this._treeSitterClient, width: "100%", marginBottom, @@ -674,13 +677,7 @@ export class MarkdownRenderable extends Renderable { }) renderable.add( - this.createMarkdownCodeRenderable( - this.getBlockquoteContent(token), - `${id}-content`, - 0, - this._linkifyMarkdownChunks, - "markup.quote", - ), + this.createMarkdownCodeRenderable(this.getBlockquoteContent(token), `${id}-content`, 0, "markup.quote"), ) return renderable @@ -917,7 +914,6 @@ export class MarkdownRenderable extends Renderable { this.normalizeScrollbackMarkdownBlockRaw(token.raw), id, 0, - this._linkifyMarkdownChunks, undefined, this.createInitialStyledText(token), ) @@ -928,14 +924,7 @@ export class MarkdownRenderable extends Renderable { if (token.type === "hr") return this.createHorizontalRuleRenderable(id) if (token.type === "table") return this.createTableBlock(token as Tokens.Table, id).renderable return token.raw - ? this.createMarkdownCodeRenderable( - token.raw, - id, - 0, - this._linkifyMarkdownChunks, - undefined, - this.createInitialStyledText(token), - ) + ? this.createMarkdownCodeRenderable(token.raw, id, 0, undefined, this.createInitialStyledText(token)) : null } @@ -1008,7 +997,6 @@ export class MarkdownRenderable extends Renderable { this.getBlockquoteContent(token), `${renderable.id}-content`, 0, - this._linkifyMarkdownChunks, "markup.quote", ), ) @@ -1510,7 +1498,6 @@ export class MarkdownRenderable extends Renderable { markdownRaw, id, 0, - this._linkifyMarkdownChunks, undefined, this.createInitialStyledText(token), ) @@ -1578,7 +1565,6 @@ export class MarkdownRenderable extends Renderable { token.raw, id, marginBottom, - this._linkifyMarkdownChunks, undefined, this.createInitialStyledText(token), ) @@ -1746,7 +1732,6 @@ export class MarkdownRenderable extends Renderable { this.getTopLevelBlockRaw(token) ?? token.raw, `${this.id}-block-${index}`, marginBottom, - this._linkifyMarkdownChunks, undefined, this.createInitialStyledText(token), ) @@ -2111,7 +2096,6 @@ export class MarkdownRenderable extends Renderable { this.getTopLevelBlockRaw(state.token) ?? state.token.raw, `${this.id}-block-${i}`, marginBottom, - this._linkifyMarkdownChunks, undefined, this.createInitialStyledText(state.token), ) diff --git a/packages/core/src/renderables/__tests__/Markdown.links.test.ts b/packages/core/src/renderables/__tests__/Markdown.links.test.ts new file mode 100644 index 000000000..dbb938914 --- /dev/null +++ b/packages/core/src/renderables/__tests__/Markdown.links.test.ts @@ -0,0 +1,175 @@ +import { afterAll, afterEach, beforeAll, expect, test } from "bun:test" +import { Buffer } from "node:buffer" +import { mkdir } from "node:fs/promises" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { RGBA } from "../../lib/RGBA.js" +import { TreeSitterClient } from "../../lib/tree-sitter/index.js" +import { CliRenderer } from "../../renderer.js" +import { SyntaxStyle } from "../../syntax-style.js" +import { createTestStdin, TestWriteStream } from "../../testing/test-streams.js" +import { CodeRenderable } from "../Code.js" +import { MarkdownRenderable } from "../Markdown.js" + +class CapturedStdout extends TestWriteStream { + readonly writes: Buffer[] = [] + override _write(chunk: Uint8Array, _encoding: BufferEncoding, callback: (error?: Error | null) => void): void { + this.writes.push(Buffer.from(chunk)) + callback() + } +} + +interface LinkSegment { + text: string + row: number + id: string + url: string +} +function linkSegments(bytes: Buffer): LinkSegment[] { + const input = bytes.toString("utf8") + const result: Array = [] + let row = 1 + let column = 1 + let id = "" + let url = "" + + for (let offset = 0; offset < input.length; ) { + if (input.startsWith("\x1b]8;", offset)) { + const end = input.indexOf("\x1b\\", offset) + const payload = input.slice(offset + 4, end) + const separator = payload.indexOf(";") + const params = separator < 0 ? payload : payload.slice(0, separator) + const target = separator < 0 ? "" : payload.slice(separator + 1) + id = params.startsWith("id=") ? params.slice(3) : "" + url = target + offset = end + 2 + continue + } + if (input.startsWith("\x1b[", offset)) { + const match = /^\x1b\[([0-9;?]*)([A-Za-z])/u.exec(input.slice(offset)) + if (!match) { + offset++ + continue + } + const values = match[1].split(";").map((value) => Number(value || 1)) + if (match[2] === "H" || match[2] === "f") [row, column] = [values[0] ?? 1, values[1] ?? 1] + if (match[2] === "A") row -= values[0] ?? 1 + if (match[2] === "B") row += values[0] ?? 1 + if (match[2] === "C") column += values[0] ?? 1 + if (match[2] === "D") column -= values[0] ?? 1 + if (match[2] === "G") column = values[0] ?? 1 + offset += match[0].length + continue + } + const char = String.fromCodePoint(input.codePointAt(offset)!) + if (url && char >= " ") { + const previous = result.at(-1) + if ( + previous && + previous.row === row && + previous.endColumn === column && + previous.id === id && + previous.url === url + ) { + previous.text += char + previous.endColumn++ + } else result.push({ text: char, row, id, url, endColumn: column + 1 }) + } + if (char >= " ") column++ + offset += char.length + } + return result.map(({ endColumn, ...segment }) => segment) +} +const syntaxStyle = SyntaxStyle.fromStyles({ default: { fg: RGBA.fromInts(255, 255, 255, 255) } }) +let treeSitterClient: TreeSitterClient +let renderer: CliRenderer | undefined +beforeAll(async () => { + const dataPath = join(tmpdir(), "tree-sitter-markdown-link-test-data") + await mkdir(dataPath, { recursive: true }) + treeSitterClient = new TreeSitterClient({ dataPath }) + await treeSitterClient.initialize() +}) +afterEach(() => renderer?.destroy()) +afterAll(() => treeSitterClient.destroy()) +async function render(content: string, width = 160, hyperlinks = true) { + const stdout = new CapturedStdout(width, 30) as CapturedStdout & NodeJS.WriteStream + renderer = new CliRenderer(createTestStdin(), stdout, width, 30, { useThread: false, remote: true }) + if (hyperlinks) (renderer as any).lib.processCapabilityResponse(renderer.rendererPtr, "\x1bP>|kitty(0.40.1)\x1b\\") + const markdown = new MarkdownRenderable(renderer, { + content, + syntaxStyle, + treeSitterClient, + tableOptions: { widthMode: "content", wrapMode: "char" }, + }) + renderer.root.add(markdown) + + for (let attempt = 0; attempt < 20; attempt++) { + await (renderer as any).loop() + const pending = markdown + .getChildren() + .filter((child): child is CodeRenderable => child instanceof CodeRenderable && child.isHighlighting) + if (pending.length === 0) break + await Promise.all(pending.map((child) => child.highlightingDone)) + } + await (renderer as any).loop() + await (renderer as any)._feed.idle() + await new Promise((resolve) => setImmediate(resolve)) + const bytes = Buffer.concat(stdout.writes) + return { bytes, links: linkSegments(bytes) } +} +const linked = (text: string, url: string) => ({ + text, + row: expect.any(Number), + id: expect.stringMatching(/^.+$/), + url, +}) +test("renders readable Markdown without OSC 8 when hyperlinks are unsupported", async () => { + const rendered = await render("[label](https://example.com) HTTPS://BARE.EXAMPLE", 160, false) + expect(rendered.links).toEqual([]) + expect(rendered.bytes.includes(Buffer.from("label"))).toBe(true) + expect(rendered.bytes.includes(Buffer.from("\x1b]8;"))).toBe(false) +}) +test("emits exact non-empty targets for labels, entities, bare URLs, and exclusion adjacency", async () => { + const rendered = await render( + "[a&b](https://example.test/?a=1&b=2) HTTPS://EXAMPLE.COM, https://safe.test`https://code.test`", + ) + expect(rendered.links).toContainEqual(linked("a&b", "https://example.test/?a=1&b=2")) + expect(rendered.links).toContainEqual(linked("HTTPS://EXAMPLE.COM", "HTTPS://EXAMPLE.COM")) + expect(rendered.links).toContainEqual(linked("https://safe.test", "https://safe.test")) + expect(rendered.links.some((link) => link.url.includes("code.test"))).toBe(false) +}) +test("preserves one non-empty id across wrapped prose and table links", async () => { + const url = "HTTPS://EXAMPLE.COM/A/VERY/LONG/PATH" + const rendered = await render(`${url}\n\n| URL |\n| --- |\n| ${url} |`, 18) + const ids = new Set(rendered.links.filter((link) => link.url === url).map((link) => link.id)) + expect(rendered.links.filter((link) => link.url === url).length).toBeGreaterThan(2) + expect(ids).toEqual(new Set([expect.stringMatching(/^.+$/)])) +}) + +test("rejects table and image controls before native OSC or control output", async () => { + const attack = "https://safe.test/\x07\x1b]8;;https://evil.test" + const encoded = "https://safe.test/]8;;https://evil.test" + const rendered = await render(`| links |\n| --- |\n| ${attack} |\n| ![image](${encoded}) |`) + expect(rendered.links.some((link) => link.url.includes("evil.test"))).toBe(false) + expect(rendered.bytes.includes(Buffer.from(attack))).toBe(false) + expect(rendered.bytes.includes(Buffer.from("\x1b]8;;https://evil.test"))).toBe(false) +}) + +test("keeps concealed entity replacement cells linked", async () => { + const rendered = await render("[a&b](https://x.test)") + expect(rendered.links).toContainEqual(linked("a&b", "https://x.test")) +}) + +test("preserves literal ampersands in link targets", async () => { + const rendered = await render( + "[escaped](https://x.test/?q=\\©) https://x.test/?a=1© [entity](https://x.test/?a=1&b=2)", + ) + expect(rendered.links).toContainEqual(linked("escaped", "https://x.test/?q=©")) + expect(rendered.links).toContainEqual(linked("https://x.test/?a=1©", "https://x.test/?a=1©")) + expect(rendered.links).toContainEqual(linked("entity", "https://x.test/?a=1&b=2")) +}) + +test("links a normal label after an escaped image marker", async () => { + const rendered = await render("\\![label](https://x.test)") + expect(rendered.links).toContainEqual(linked("label", "https://x.test")) +}) diff --git a/packages/web/src/content/docs/components/code.mdx b/packages/web/src/content/docs/components/code.mdx index 92693b19a..ef130a215 100644 --- a/packages/web/src/content/docs/components/code.mdx +++ b/packages/web/src/content/docs/components/code.mdx @@ -181,6 +181,25 @@ const code = new CodeRenderable(renderer, { }) ``` +## Highlight hooks + +Use `onHighlight` to adjust source ranges before chunks are created. It can also set source-ordered, non-overlapping `context.linkRanges` when a language-specific integration provides hyperlink targets. Range starts are inclusive and ends are exclusive. Use `onChunks` for transformations that must run after highlighting and concealment. Returning `undefined` from either callback keeps the input unchanged. + +```typescript +const code = new CodeRenderable(renderer, { + content: "const answer = 42", + filetype: "typescript", + syntaxStyle, + onHighlight: (highlights, context) => { + context.linkRanges = [{ start: 6, end: 12, url: "https://example.com/answer" }] + return highlights + }, + onChunks: (chunks) => chunks, +}) +``` + +`CodeRenderable` does not infer terminal hyperlinks from a filetype. `MarkdownRenderable` owns Markdown link detection; custom `CodeRenderable` integrations can add link metadata through `onHighlight` source ranges. + ## With line numbers Use `LineNumberRenderable` to add line numbers: @@ -223,15 +242,18 @@ renderer.root.add(scrollbox) ## Properties -| Property | Type | Default | Description | -| ------------------ | ------------------ | -------- | ---------------------------------------- | -| `content` | `string` | `""` | Source code to display | -| `filetype` | `string` | - | Language for syntax highlighting | -| `syntaxStyle` | `SyntaxStyle` | required | Syntax highlighting theme | -| `streaming` | `boolean` | `false` | Optimize for incremental content updates | -| `conceal` | `boolean` | `true` | Hide concealed syntax elements | -| `drawUnstyledText` | `boolean` | `true` | Show text before highlighting completes | -| `treeSitterClient` | `TreeSitterClient` | - | Custom Tree-sitter client instance | +| Property | Type | Default | Description | +| ------------------ | --------------------- | -------- | ------------------------------------------------- | +| `content` | `string` | `""` | Source code to display | +| `filetype` | `string` | - | Language for syntax highlighting | +| `syntaxStyle` | `SyntaxStyle` | required | Syntax highlighting theme | +| `streaming` | `boolean` | `false` | Optimize for incremental content updates | +| `conceal` | `boolean` | `true` | Hide concealed syntax elements | +| `drawUnstyledText` | `boolean` | `true` | Show text before highlighting completes | +| `treeSitterClient` | `TreeSitterClient` | - | Custom Tree-sitter client instance | +| `baseHighlight` | `string` | - | Base scope merged into generated chunks | +| `onHighlight` | `OnHighlightCallback` | - | Transform source highlights before chunk creation | +| `onChunks` | `OnChunksCallback` | - | Transform generated chunks before display | ### Inherited from TextBufferRenderable diff --git a/packages/web/src/content/docs/components/markdown.mdx b/packages/web/src/content/docs/components/markdown.mdx index e1fb66a93..c4f502f53 100644 --- a/packages/web/src/content/docs/components/markdown.mdx +++ b/packages/web/src/content/docs/components/markdown.mdx @@ -66,6 +66,19 @@ const markdown = new MarkdownRenderable(renderer, { Use `concealCode` to control concealment inside fenced code blocks independently (`false` by default). +## Terminal hyperlinks + +Markdown link labels, their visible destinations, and bare HTTP(S) URLs carry OSC 8 hyperlink metadata. Inline code, fenced code, and raw HTML are not scanned for bare URLs. + +The terminal controls interaction, such as Cmd-click or Ctrl-click. When the terminal does not support hyperlinks, the same readable text is rendered without OSC 8 output. + +```typescript +const markdown = new MarkdownRenderable(renderer, { + content: "[OpenTUI](https://opentui.com) or https://opentui.com/docs", + syntaxStyle, +}) +``` + ## Streaming updates Enable streaming mode for incremental updates. Keep it `true` while appending chunks, then set `markdown.streaming = false` when complete to finalize trailing block parsing. Tables include trailing partial rows, with missing cells rendered empty.