From 496501a5cd2003f0581d4e26c091d70d90849a2e Mon Sep 17 00:00:00 2001 From: Emanuele Stoppa Date: Thu, 6 Aug 2026 15:43:20 +0100 Subject: [PATCH 1/2] refactor(md/parse): two-phase parse --- .../tests/specs/markdown/reference_link.md | 4 + .../specs/markdown/reference_link.md.snap | 8 + .../biome_markdown_parser/src/inline_phase.rs | 179 ++++++++++++ crates/biome_markdown_parser/src/lexer/mod.rs | 64 +++-- crates/biome_markdown_parser/src/lib.rs | 24 +- .../src/link_reference.rs | 40 --- crates/biome_markdown_parser/src/parser.rs | 269 ++++++++++++++++-- .../src/syntax/header.rs | 18 +- .../src/syntax/link_block.rs | 4 + .../biome_markdown_parser/src/syntax/mod.rs | 18 +- crates/biome_markdown_parser/src/to_html.rs | 12 + .../biome_markdown_parser/src/token_source.rs | 13 +- .../tests/cst_invariants.rs | 41 +++ crates/biome_parser/src/diagnostic.rs | 4 + 14 files changed, 604 insertions(+), 94 deletions(-) create mode 100644 crates/biome_markdown_parser/src/inline_phase.rs delete mode 100644 crates/biome_markdown_parser/src/link_reference.rs diff --git a/crates/biome_markdown_formatter/tests/specs/markdown/reference_link.md b/crates/biome_markdown_formatter/tests/specs/markdown/reference_link.md index b54141e709ed..19908018f9bc 100644 --- a/crates/biome_markdown_formatter/tests/specs/markdown/reference_link.md +++ b/crates/biome_markdown_formatter/tests/specs/markdown/reference_link.md @@ -14,6 +14,10 @@ text with [inline][ref] in the middle [link with **bold**][ref] +# *[heading*][ref] + +> - [nested][ref] + [world]: https://example.com "World" [ label ]: https://example.com "Label" [*foo* bar]: https://example.com "Foo Bar" diff --git a/crates/biome_markdown_formatter/tests/specs/markdown/reference_link.md.snap b/crates/biome_markdown_formatter/tests/specs/markdown/reference_link.md.snap index 41092cbff109..01685d84fb7e 100644 --- a/crates/biome_markdown_formatter/tests/specs/markdown/reference_link.md.snap +++ b/crates/biome_markdown_formatter/tests/specs/markdown/reference_link.md.snap @@ -22,6 +22,10 @@ text with [inline][ref] in the middle [link with **bold**][ref] +# *[heading*][ref] + +> - [nested][ref] + [world]: https://example.com "World" [ label ]: https://example.com "Label" [*foo* bar]: https://example.com "Foo Bar" @@ -51,6 +55,10 @@ text with [inline][ref] in the middle [link with **bold**][ref] +# *[heading*][ref] + +> - [nested][ref] + [world]: https://example.com "World" [ label ]: https://example.com "Label" [*foo* bar]: https://example.com "Foo Bar" diff --git a/crates/biome_markdown_parser/src/inline_phase.rs b/crates/biome_markdown_parser/src/inline_phase.rs new file mode 100644 index 000000000000..202b0a4d8c10 --- /dev/null +++ b/crates/biome_markdown_parser/src/inline_phase.rs @@ -0,0 +1,179 @@ +//! Completes inline syntax after Markdown block parsing. +//! +//! CommonMark separates parsing into a block phase followed by an inline phase +//! because link reference definitions are document-global and are not all known +//! until block parsing finishes. See the +//! [CommonMark parsing strategy](https://spec.commonmark.org/0.31.2/#appendix-a-parsing-strategy). +//! +//! The block parser records each inline event subtree together with its source +//! range and the number of definitions known at that point. Inline subtrees that +//! precede later definitions are parsed again with the complete definition index. +//! Their replacement events, trivia, and diagnostics are merged before the +//! lossless tree sink constructs the CST, so source text is never rewritten. + +use biome_markdown_syntax::MarkdownSyntaxKind::MD_ROOT; +use biome_parser::Parser; +use biome_parser::diagnostic::merge_diagnostics; +use biome_parser::event::Event; +use biome_parser::token_source::Trivia; +use biome_rowan::{TextRange, TextSize}; + +use crate::MarkdownParserOptions; +use crate::parser::{ + DeferredInline, DeferredInlineFlavor, LinkReferenceDefinitions, MarkdownParser, + MarkdownParserOutput, +}; +use crate::syntax::header::parse_header_content; +use crate::syntax::parse_inline_item_list; + +/// Resolves deferred inline subtrees and returns an event stream ready for the +/// lossless tree sink. +/// +/// The returned output contains no deferred inline records. Event replacement +/// requires the recorded event ranges to be ordered and non-overlapping. +pub(crate) fn parse_deferred_inlines( + source: &str, + options: &MarkdownParserOptions, + mut output: MarkdownParserOutput, +) -> MarkdownParserOutput { + debug_assert!(output.events.iter().all(|event| { + !matches!( + event, + Event::Start { + forward_parent: Some(_), + .. + } + ) + })); + + let mut replacements = Vec::new(); + let mut previous_event_end = 0; + + for deferred in &output.deferred_inlines { + if deferred.definitions_len == output.link_reference_definitions.len() { + continue; + } + if deferred.event_range.start < previous_event_end + || deferred.event_range.start > deferred.event_range.end + || deferred.event_range.end > output.events.len() + { + continue; + } + + let Some(fragment) = parse_inline_fragment( + source, + deferred, + options.clone(), + &output.link_reference_definitions, + ) else { + continue; + }; + + replacements.push((deferred, fragment)); + previous_event_end = deferred.event_range.end; + } + + let mut events = Vec::with_capacity(output.events.len()); + let mut old_events = std::mem::take(&mut output.events).into_iter(); + let mut inline_trivia = Vec::new(); + let mut inline_diagnostics = Vec::new(); + let mut reparsed_ranges = Vec::with_capacity(replacements.len()); + let mut event_position = 0; + + for (deferred, fragment) in replacements { + debug_assert!(event_position <= deferred.event_range.start); + while event_position < deferred.event_range.start { + let Some(event) = old_events.next() else { + break; + }; + events.push(event); + event_position += 1; + } + while event_position < deferred.event_range.end { + if old_events.next().is_none() { + break; + } + event_position += 1; + } + events.extend(fragment.events); + + reparsed_ranges.push(deferred.source_range); + inline_trivia.extend(fragment.trivia); + inline_diagnostics.extend(fragment.diagnostics); + } + events.extend(old_events); + output.events = events; + + let mut range_index = 0; + output.trivia.retain(|trivia| { + !overlaps_ordered_ranges(trivia.text_range(), &reparsed_ranges, &mut range_index) + }); + output.trivia.extend(inline_trivia); + output.trivia.sort_by_key(Trivia::offset); + + range_index = 0; + output.diagnostics.retain(|diagnostic| { + diagnostic + .span() + .is_none_or(|span| !overlaps_ordered_ranges(span, &reparsed_ranges, &mut range_index)) + }); + output.diagnostics = merge_diagnostics(output.diagnostics, inline_diagnostics); + output.deferred_inlines.clear(); + + output +} + +fn parse_inline_fragment<'source>( + source: &'source str, + deferred: &DeferredInline, + options: MarkdownParserOptions, + definitions: &'source LinkReferenceDefinitions, +) -> Option { + let source_end = TextSize::try_from(source.len()).ok()?; + let mut parser = MarkdownParser::new_range( + source, + TextRange::new(deferred.source_range.start(), source_end), + options, + deferred.context, + definitions, + )?; + + let wrapper = parser.start(); + + match deferred.flavor { + DeferredInlineFlavor::Paragraph => parse_inline_item_list(&mut parser), + DeferredInlineFlavor::AtxParagraph => parse_header_content(&mut parser), + } + + debug_assert_eq!( + parser.cur_range().start(), + deferred.source_range.end(), + "inline fragment {:?} stopped at {:?} ({:?})", + deferred.source_range, + parser.cur_range(), + (parser.cur(), &source[deferred.source_range]) + ); + wrapper.complete(&mut parser, MD_ROOT); + let mut output = parser.finish(); + output.events.remove(0); + output.events.pop(); + Some(output) +} +fn overlaps_ordered_ranges( + range: TextRange, + ranges: &[TextRange], + range_index: &mut usize, +) -> bool { + while ranges + .get(*range_index) + .is_some_and(|candidate| candidate.end() <= range.start()) + { + *range_index += 1; + } + + ranges.get(*range_index).is_some_and(|candidate| { + candidate + .intersect(range) + .is_some_and(|intersection| !intersection.is_empty()) + }) +} diff --git a/crates/biome_markdown_parser/src/lexer/mod.rs b/crates/biome_markdown_parser/src/lexer/mod.rs index 4c13111188b1..0689262d7326 100644 --- a/crates/biome_markdown_parser/src/lexer/mod.rs +++ b/crates/biome_markdown_parser/src/lexer/mod.rs @@ -9,7 +9,7 @@ use biome_parser::diagnostic::ParseDiagnostic; use biome_parser::lexer::{ LexContext, Lexer, LexerCheckpoint, LexerWithCheckpoint, ReLexer, TokenFlags, }; -use biome_rowan::{SyntaxKind, TextSize}; +use biome_rowan::{SyntaxKind, TextRange, TextSize}; use biome_unicode_table::Dispatch::{self, AMP, *}; use biome_unicode_table::{is_unicode_punctuation, lookup_byte}; @@ -90,6 +90,9 @@ pub(crate) struct MarkdownLexer<'src> { /// Source text source: &'src str, + /// Exclusive end of the source range visible to this lexer. + end: usize, + /// The start byte position in the source text of the next token. position: usize, @@ -210,6 +213,15 @@ impl<'src> Lexer<'src> for MarkdownLexer<'src> { self.position } + fn is_eof(&self) -> bool { + self.position >= self.end + } + + fn byte_at(&self, offset: usize) -> Option { + let position = self.position.checked_add(offset)?; + (position < self.end).then(|| self.source.as_bytes()[position]) + } + fn push_diagnostic(&mut self, diagnostic: ParseDiagnostic) { self.diagnostics.push(diagnostic); } @@ -230,15 +242,31 @@ impl<'src> Lexer<'src> for MarkdownLexer<'src> { impl<'src> MarkdownLexer<'src> { /// Make a new lexer from a str, this is safe because strs are valid utf8 pub fn from_str(source: &'src str) -> Self { + Self::from_valid_range(source, TextSize::from(0), source.len()) + } + + pub fn from_range(source: &'src str, range: TextRange) -> Option { + let start: usize = range.start().into(); + let end: usize = range.end().into(); + source.get(start..end)?; + + Some(Self::from_valid_range(source, range.start(), end)) + } + + fn from_valid_range(source: &'src str, start: TextSize, end: usize) -> Self { Self { source, - // Start of document is treated as start of line for indentation purposes - after_newline: true, + end, + after_newline: start == TextSize::from(0) + || matches!( + source.as_bytes().get(usize::from(start) - 1), + Some(b'\n' | b'\r') + ), unicode_bom_length: 0, current_kind: TOMBSTONE, - current_start: TextSize::from(0), + current_start: start, current_flags: TokenFlags::empty(), - position: 0, + position: start.into(), diagnostics: vec![], force_ordered_list_marker: false, relex_span: None, @@ -249,6 +277,10 @@ impl<'src> MarkdownLexer<'src> { self.force_ordered_list_marker = value; } + pub fn range_end(&self) -> usize { + self.end + } + /// Sets the target for the next [MarkdownReLexContext::Span] re-lex. pub fn set_relex_span(&mut self, end: usize, kind: MarkdownSyntaxKind) { self.relex_span = Some((end, kind)); @@ -589,10 +621,8 @@ impl<'src> MarkdownLexer<'src> { /// Returns the byte at position `self.position + offset` or `None` if it is out of bounds. #[inline] fn byte_at(&self, offset: usize) -> Option { - self.source() - .as_bytes() - .get(self.position() + offset) - .copied() + let position = self.position.checked_add(offset)?; + (position < self.end).then(|| self.source.as_bytes()[position]) } /// Peeks at the next byte @@ -1221,7 +1251,7 @@ impl<'src> MarkdownLexer<'src> { /// Returns `true` if the parser is at or passed the end of the file. #[inline] fn is_eof(&self) -> bool { - self.position >= self.source.len() + self.position >= self.end } /// Consume textual characters until hitting special markdown syntax. @@ -1385,7 +1415,7 @@ impl<'src> MarkdownLexer<'src> { } fn is_ordered_list_marker_at(&self, idx: usize) -> bool { - if idx >= self.source.len() { + if idx >= self.end { return false; } @@ -1393,7 +1423,7 @@ impl<'src> MarkdownLexer<'src> { let mut pos = idx; let mut digit_count = 0; - while pos < bytes.len() && bytes[pos].is_ascii_digit() { + while pos < self.end && bytes[pos].is_ascii_digit() { digit_count += 1; if digit_count > MAX_ORDERED_LIST_MARKER_DIGITS { return false; @@ -1405,12 +1435,12 @@ impl<'src> MarkdownLexer<'src> { return false; } - if pos >= bytes.len() || !(bytes[pos] == b'.' || bytes[pos] == b')') { + if pos >= self.end || !(bytes[pos] == b'.' || bytes[pos] == b')') { return false; } pos += 1; - if pos >= bytes.len() { + if pos >= self.end { return true; } @@ -1423,7 +1453,7 @@ impl<'src> MarkdownLexer<'src> { fn is_at_trailing_hash_closing_whitespace(&self) -> bool { let mut i = self.position; let bytes = self.source.as_bytes(); - let len = bytes.len(); + let len = self.end; let mut saw_ws = false; while i < len { @@ -1526,7 +1556,7 @@ impl<'src> MarkdownLexer<'src> { while let Some(b'_') = self.byte_at(offset) { offset += 1; } - let after = self.source[self.position + offset..].chars().next(); + let after = self.source[self.position + offset..self.end].chars().next(); is_word_char(after) } @@ -1626,7 +1656,7 @@ impl<'src> ReLexer<'src> for MarkdownLexer<'src> { .relex_span .take() .expect("set_relex_span must be called before a Span re-lex"); - debug_assert!(end > self.position && end <= self.source.len()); + debug_assert!(end > self.position && end <= self.end); self.position = end; kind } diff --git a/crates/biome_markdown_parser/src/lib.rs b/crates/biome_markdown_parser/src/lib.rs index a2edccb300d0..4733dc793916 100644 --- a/crates/biome_markdown_parser/src/lib.rs +++ b/crates/biome_markdown_parser/src/lib.rs @@ -7,8 +7,8 @@ use biome_rowan::{AstNode, NodeCache}; use parser::MarkdownParser; use syntax::parse_document; +mod inline_phase; mod lexer; -mod link_reference; mod parser; mod syntax; mod token_source; @@ -38,26 +38,26 @@ pub fn parse_markdown_with_cache( cache: &mut NodeCache, options: MarkdownParserOptions, ) -> MarkdownParse { - let link_definitions = - link_reference::collect_link_reference_definitions(source, options.clone()); - let mut parser = MarkdownParser::new(source, options); - parser.set_link_reference_definitions(link_definitions); + let mut parser = MarkdownParser::new(source, options.clone()); parse_document(&mut parser); - let (events, diagnostics, trivia, list_tightness, list_item_indents, quote_indents) = - parser.finish(); + let output = inline_phase::parse_deferred_inlines(source, &options, parser.finish()); + debug_assert!( + output.deferred_inlines.is_empty(), + "There shouldn't be deferred inline nodes." + ); - let mut tree_sink = MarkdownLosslessTreeSink::with_cache(source, &trivia, cache); - biome_parser::event::process(&mut tree_sink, events, diagnostics); + let mut tree_sink = MarkdownLosslessTreeSink::with_cache(source, &output.trivia, cache); + biome_parser::event::process(&mut tree_sink, output.events, output.diagnostics); let (green, diagnostics) = tree_sink.finish(); MarkdownParse::new( green, diagnostics, - list_tightness, - list_item_indents, - quote_indents, + output.list_tightness, + output.list_item_indents, + output.quote_indents, ) } diff --git a/crates/biome_markdown_parser/src/link_reference.rs b/crates/biome_markdown_parser/src/link_reference.rs deleted file mode 100644 index 10a162bd3113..000000000000 --- a/crates/biome_markdown_parser/src/link_reference.rs +++ /dev/null @@ -1,40 +0,0 @@ -use std::collections::HashSet; - -use biome_markdown_syntax::MdLinkReferenceDefinition; -use biome_rowan::AstNode; - -use crate::MarkdownLosslessTreeSink; -use crate::MarkdownParserOptions; -use crate::parser::MarkdownParser; -use crate::syntax::parse_document; -use crate::syntax::reference::normalize_reference_label; - -pub(crate) fn collect_link_reference_definitions( - source: &str, - options: MarkdownParserOptions, -) -> HashSet { - let mut parser = MarkdownParser::new(source, options); - parse_document(&mut parser); - let (events, diagnostics, trivia, _list_tightness, _list_item_indents, _quote_indents) = - parser.finish(); - - let mut tree_sink = MarkdownLosslessTreeSink::new(source, &trivia); - biome_parser::event::process(&mut tree_sink, events, diagnostics); - let (root, _) = tree_sink.finish(); - - let mut definitions = HashSet::new(); - - for node in root.descendants() { - if let Some(def) = MdLinkReferenceDefinition::cast(node) - && let Ok(label) = def.label() - { - let raw = label.syntax().text_trimmed().to_string(); - let normalized = normalize_reference_label(&raw); - if !normalized.is_empty() { - definitions.insert(normalized.into_owned()); - } - } - } - - definitions -} diff --git a/crates/biome_markdown_parser/src/parser.rs b/crates/biome_markdown_parser/src/parser.rs index 34a765488990..b76f211d1deb 100644 --- a/crates/biome_markdown_parser/src/parser.rs +++ b/crates/biome_markdown_parser/src/parser.rs @@ -6,7 +6,9 @@ use biome_parser::token_source::Trivia; use biome_parser::{ParserContextCheckpoint, diagnostic::merge_diagnostics}; use biome_rowan::{TextRange, TextSize}; use std::cell::Cell; -use std::collections::HashSet; +use std::collections::HashMap; +use std::hash::{DefaultHasher, Hash, Hasher}; +use std::ops::Range; use crate::lexer::{MarkdownLexContext, MarkdownReLexContext}; use crate::syntax::TAB_STOP_SPACES; @@ -72,8 +74,10 @@ pub(crate) struct MarkdownParserState { pub(crate) list_item_ordered_delim: Option, /// Emphasis parsing context for the current inline item list. pub(crate) emphasis_context: Option, - /// Normalized link reference definitions collected in a prepass. - pub(crate) link_reference_definitions: HashSet, + /// Link reference definitions accepted while parsing block structure. + pub(crate) link_reference_definitions: LinkReferenceDefinitions, + /// Inline event subtrees reparsed after all definitions are known. + pub(crate) deferred_inlines: Vec, /// Whether a following non-blank line should be treated as paragraph /// continuation after a link reference definition. pub(crate) link_reference_definition_continuation: bool, @@ -106,14 +110,142 @@ pub struct ListItemIndent { pub spaces_after_marker: usize, } -type FinishResult = ( - Vec>, - Vec, - Vec, - Vec, - Vec, - Vec, -); +/// Products of a Markdown parse before construction of the green tree. +/// +/// `deferred_inlines` identifies provisional inline event subtrees that may +/// depend on definitions discovered later in the document. The inline phase +/// resolves those records before `events`, `trivia`, and `diagnostics` are sent +/// to the lossless tree sink. +pub(crate) struct MarkdownParserOutput { + /// Syntax events in source order. + pub(crate) events: Vec>, + /// Lexer and parser diagnostics in source order. + pub(crate) diagnostics: Vec, + /// Trivia associated with the event stream. + pub(crate) trivia: Vec, + /// Provisional inline event subtrees awaiting document-global resolution. + pub(crate) deferred_inlines: Vec, + /// Link reference definitions accepted while parsing block structure. + pub(crate) link_reference_definitions: LinkReferenceDefinitions, + /// Tightness metadata keyed by final list ranges. + pub(crate) list_tightness: Vec, + /// Indentation metadata keyed by final list-item ranges. + pub(crate) list_item_indents: Vec, + /// Indentation metadata keyed by final quote ranges. + pub(crate) quote_indents: Vec, +} + +/// CST wrapper required when rebuilding a deferred inline event subtree. +#[derive(Debug, Clone, Copy)] +pub(crate) enum DeferredInlineFlavor { + /// An `MdInlineItemList` that is already enclosed by its block node. + Paragraph, + /// An ATX heading's `MdParagraph`, including its inline item list. + AtxParagraph, +} + +/// Identifies a provisional inline subtree and the parser state needed to +/// rebuild it after all link reference definitions are known. +/// +/// `event_range` and `source_range` describe the same inline subtree in the +/// parser event stream and original source respectively. Event ranges are +/// ordered and non-overlapping. +#[derive(Debug, Clone)] +pub(crate) struct DeferredInline { + /// Half-open range of provisional events replaced by the inline phase. + pub(crate) event_range: Range, + /// Original source covered by the provisional inline subtree. + pub(crate) source_range: TextRange, + /// Wrapper shape expected at the replacement site. + pub(crate) flavor: DeferredInlineFlavor, + /// Block-container state at the start of the inline subtree. + pub(crate) context: InlineContainerContext, + /// Number of definitions known when the provisional subtree was parsed. + /// + /// A smaller value than the final definition count means later definitions + /// may change reference and emphasis parsing in this subtree. + pub(crate) definitions_len: usize, +} + +/// Block-container state that affects parsing of a detached inline source range. +/// +/// Inline reparsing starts at the original absolute source offset but outside +/// the recursive block parser. This snapshot restores the quote, list, and +/// virtual-line context needed to classify continuation prefixes and indentation +/// exactly as they appear in the final CST. +#[derive(Debug, Clone, Copy)] +pub(crate) struct InlineContainerContext { + /// Active block quote nesting at the start of the inline range. + block_quote_depth: usize, + /// Active list nesting at the start of the inline range. + list_nesting_depth: usize, + /// Columns required for a line to continue the current list item. + list_item_required_indent: usize, + /// Column where the current list marker starts. + list_item_marker_indent: usize, + /// Bullet marker of the containing unordered list, when present. + list_item_marker_kind: Option, + /// Delimiter of the containing ordered list, when present. + list_item_ordered_delim: Option, + /// Logical line start after a container prefix has been consumed. + virtual_line_start: Option, +} + +/// Index of link reference definitions accepted by the block parser. +/// +/// Definition labels remain in the source and are represented by ranges. The +/// hash index uses CommonMark's normalized-label equivalence only to narrow +/// lookup candidates; it never replaces the original label text. +#[derive(Debug, Default)] +pub(crate) struct LinkReferenceDefinitions { + /// Source ranges of definition labels in document order. + ranges: Vec, + /// Normalized-label hash to indices in `ranges`. + /// + /// Each hash can identify multiple ranges because duplicate definitions and + /// hash collisions are both possible. Lookup compares normalized source text + /// from every candidate range before reporting a match. + ranges_by_hash: HashMap>, +} + +impl LinkReferenceDefinitions { + fn insert(&mut self, range: TextRange, hash: u64) { + let index = self.ranges.len(); + self.ranges.push(range); + self.ranges_by_hash.entry(hash).or_default().push(index); + } + + pub(crate) fn len(&self) -> usize { + self.ranges.len() + } + + fn contains(&self, source: &str, normalized_label: &str) -> bool { + let hash = normalized_label_hash(normalized_label); + self.ranges_by_hash + .get(&hash) + .into_iter() + .flatten() + .filter_map(|index| self.ranges.get(*index)) + .filter_map(|range| source.get(usize::from(range.start())..usize::from(range.end()))) + .any(|label| { + crate::syntax::reference::normalize_reference_label(label) == normalized_label + }) + } + + fn truncate(&mut self, len: usize, hashes: impl IntoIterator) { + self.ranges.truncate(len); + self.ranges_by_hash.clear(); + for (index, hash) in hashes { + self.ranges_by_hash.entry(hash).or_default().push(index); + } + } +} + +fn normalized_label_hash(label: &str) -> u64 { + let mut hasher = DefaultHasher::new(); + crate::syntax::reference::normalize_reference_label(label).hash(&mut hasher); + hasher.finish() +} #[derive(Debug, Clone, PartialEq, Eq)] pub struct QuoteIndent { @@ -125,6 +257,7 @@ pub(crate) struct MarkdownParser<'source> { context: ParserContext, source: MarkdownTokenSource<'source>, options: MarkdownParserOptions, + known_link_reference_definitions: Option<&'source LinkReferenceDefinitions>, state: MarkdownParserState, /// Single-entry memo for `absolute_column_at` queries. Most callers ask /// for the column of the current token offset many times in a row while @@ -139,11 +272,40 @@ impl<'source> MarkdownParser<'source> { context: ParserContext::default(), source: MarkdownTokenSource::from_str(source), options, + known_link_reference_definitions: None, state: MarkdownParserState::default(), abs_col_cache: Cell::new(None), } } + pub(crate) fn new_range( + source: &'source str, + range: TextRange, + options: MarkdownParserOptions, + context: InlineContainerContext, + definitions: &'source LinkReferenceDefinitions, + ) -> Option { + let state = MarkdownParserState { + block_quote_depth: context.block_quote_depth, + list_nesting_depth: context.list_nesting_depth, + list_item_required_indent: context.list_item_required_indent, + list_item_marker_indent: context.list_item_marker_indent, + list_item_marker_kind: context.list_item_marker_kind, + list_item_ordered_delim: context.list_item_ordered_delim, + virtual_line_start: context.virtual_line_start, + ..MarkdownParserState::default() + }; + + Some(Self { + context: ParserContext::default(), + source: MarkdownTokenSource::from_range(source, range)?, + options, + known_link_reference_definitions: Some(definitions), + state, + abs_col_cache: Cell::new(None), + }) + } + /// Cached wrapper around [`absolute_column_at`]. Returns the column of /// `offset` in the parser source, reusing the previous result when the /// offset matches. Callers like `line_start_leading_indent` and @@ -189,14 +351,46 @@ impl<'source> MarkdownParser<'source> { std::mem::replace(&mut self.state.emphasis_context, context) } - /// Replace the set of normalized link reference definitions. - pub(crate) fn set_link_reference_definitions(&mut self, definitions: HashSet) { - self.state.link_reference_definitions = definitions; + pub(crate) fn record_link_reference_definition(&mut self, label: TextRange) { + let Some(label_text) = self + .source + .source_text() + .get(usize::from(label.start())..usize::from(label.end())) + else { + return; + }; + let hash = normalized_label_hash(label_text); + self.state.link_reference_definitions.insert(label, hash); } /// Returns true if a normalized label has a link reference definition. pub(crate) fn has_link_reference_definition(&self, label: &str) -> bool { - self.state.link_reference_definitions.contains(label) + self.known_link_reference_definitions + .is_some_and(|definitions| definitions.contains(self.source.source_text(), label)) + || self + .state + .link_reference_definitions + .contains(self.source.source_text(), label) + } + + pub(crate) fn inline_container_context(&self) -> InlineContainerContext { + InlineContainerContext { + block_quote_depth: self.state.block_quote_depth, + list_nesting_depth: self.state.list_nesting_depth, + list_item_required_indent: self.state.list_item_required_indent, + list_item_marker_indent: self.state.list_item_marker_indent, + list_item_marker_kind: self.state.list_item_marker_kind, + list_item_ordered_delim: self.state.list_item_ordered_delim, + virtual_line_start: self.state.virtual_line_start, + } + } + + pub(crate) fn record_deferred_inline(&mut self, deferred: DeferredInline) { + self.state.deferred_inlines.push(deferred); + } + + pub(crate) fn link_reference_definitions_len(&self) -> usize { + self.state.link_reference_definitions.len() } /// Record tight/loose information for a parsed list node. @@ -362,6 +556,8 @@ impl<'source> MarkdownParser<'source> { MarkdownParserCheckpoint { context: self.context.checkpoint(), source: self.source.checkpoint(), + deferred_inlines_len: self.state.deferred_inlines.len(), + link_reference_definitions_len: self.state.link_reference_definitions.len(), } } @@ -610,10 +806,35 @@ impl<'source> MarkdownParser<'source> { } pub fn rewind(&mut self, checkpoint: MarkdownParserCheckpoint) { - let MarkdownParserCheckpoint { context, source } = checkpoint; + let MarkdownParserCheckpoint { + context, + source, + deferred_inlines_len, + link_reference_definitions_len, + } = checkpoint; self.context.rewind(context); self.source.rewind(source); + self.state.deferred_inlines.truncate(deferred_inlines_len); + let hashes = self + .state + .link_reference_definitions + .ranges + .iter() + .take(link_reference_definitions_len) + .enumerate() + .filter_map(|range| { + let (index, range) = range; + self.source + .source_text() + .get(usize::from(range.start())..usize::from(range.end())) + .map(|label| (index, label)) + }) + .map(|(index, label)| (index, normalized_label_hash(label))) + .collect::>(); + self.state + .link_reference_definitions + .truncate(link_reference_definitions_len, hashes); } /// Execute a lookahead operation without consuming tokens. @@ -643,20 +864,22 @@ impl<'source> MarkdownParser<'source> { result } - pub fn finish(self) -> FinishResult { + pub fn finish(self) -> MarkdownParserOutput { let (trivia, lexer_diagnostics) = self.source.finish(); let (events, parse_diagnostics) = self.context.finish(); let diagnostics = merge_diagnostics(lexer_diagnostics, parse_diagnostics); - ( + MarkdownParserOutput { events, diagnostics, trivia, - self.state.list_tightness, - self.state.list_item_indents, - self.state.quote_indents, - ) + deferred_inlines: self.state.deferred_inlines, + link_reference_definitions: self.state.link_reference_definitions, + list_tightness: self.state.list_tightness, + list_item_indents: self.state.list_item_indents, + quote_indents: self.state.quote_indents, + } } } @@ -717,4 +940,6 @@ impl<'source> Parser for MarkdownParser<'source> { pub struct MarkdownParserCheckpoint { pub(super) context: ParserContextCheckpoint, pub(super) source: MarkdownTokenSourceCheckpoint, + deferred_inlines_len: usize, + link_reference_definitions_len: usize, } diff --git a/crates/biome_markdown_parser/src/syntax/header.rs b/crates/biome_markdown_parser/src/syntax/header.rs index e92a022624a4..f9973a279ab9 100644 --- a/crates/biome_markdown_parser/src/syntax/header.rs +++ b/crates/biome_markdown_parser/src/syntax/header.rs @@ -24,7 +24,9 @@ //! --------- //! ``` -use crate::parser::{MarkdownParser, MarkdownParserCheckpoint}; +use crate::parser::{ + DeferredInline, DeferredInlineFlavor, MarkdownParser, MarkdownParserCheckpoint, +}; use crate::syntax::MAX_BLOCK_PREFIX_INDENT; use crate::syntax::inline::EmphasisContext; use crate::syntax::parse_any_inline; @@ -35,6 +37,7 @@ use biome_parser::{ Parser, prelude::ParsedSyntax::{self, *}, }; +use biome_rowan::TextRange; /// Maximum number of `#` characters allowed in an ATX heading (CommonMark ยง4.2). const MAX_HEADER_HASHES: usize = 6; @@ -171,6 +174,10 @@ pub(crate) fn parse_header_content(p: &mut MarkdownParser) { let prev_context = set_header_emphasis_context(p); // Parse content as a paragraph containing inline items + let event_start = p.context().events().len(); + let source_start = p.cur_range().start(); + let context = p.inline_container_context(); + let definitions_len = p.link_reference_definitions_len(); let m = p.start(); let inline_m = p.start(); @@ -214,6 +221,15 @@ pub(crate) fn parse_header_content(p: &mut MarkdownParser) { inline_m.complete(p, MD_INLINE_ITEM_LIST); m.complete(p, MD_PARAGRAPH); + let event_end = p.context().events().len(); + + p.record_deferred_inline(DeferredInline { + event_range: event_start..event_end, + source_range: TextRange::new(source_start, p.cur_range().start()), + flavor: DeferredInlineFlavor::AtxParagraph, + context, + definitions_len, + }); // Restore previous emphasis context p.set_emphasis_context(prev_context); diff --git a/crates/biome_markdown_parser/src/syntax/link_block.rs b/crates/biome_markdown_parser/src/syntax/link_block.rs index 322ad7df0528..ae433eeb9fc3 100644 --- a/crates/biome_markdown_parser/src/syntax/link_block.rs +++ b/crates/biome_markdown_parser/src/syntax/link_block.rs @@ -23,6 +23,7 @@ use biome_markdown_syntax::MarkdownSyntaxKind::*; use biome_parser::Parser; use biome_parser::prelude::ParsedSyntax::{self, *}; +use biome_rowan::TextRange; use crate::MarkdownParser; use crate::lexer::MarkdownLexContext; @@ -396,7 +397,10 @@ pub(crate) fn parse_link_block(p: &mut MarkdownParser) -> ParsedSyntax { p.expect(L_BRACK); // Label - parse until ] + let label_start = p.cur_range().start(); parse_link_label(p); + let label_end = p.cur_range().start(); + p.record_link_reference_definition(TextRange::new(label_start, label_end)); // ] - closing bracket p.expect(R_BRACK); diff --git a/crates/biome_markdown_parser/src/syntax/mod.rs b/crates/biome_markdown_parser/src/syntax/mod.rs index 80994afa90b7..eb9c34906b13 100644 --- a/crates/biome_markdown_parser/src/syntax/mod.rs +++ b/crates/biome_markdown_parser/src/syntax/mod.rs @@ -42,7 +42,7 @@ use biome_parser::{ prelude::ParsedSyntax::{self, *}, token_set, }; -use biome_rowan::TextSize; +use biome_rowan::{TextRange, TextSize}; use fenced_code_block::{ at_fenced_code_block, info_string_has_backtick, parse_fenced_code_block, parse_fenced_code_block_force, @@ -64,6 +64,7 @@ use thematic_break_block::{at_thematic_break_block, parse_thematic_break_block}; use crate::MarkdownParser; use crate::lexer::MarkdownReLexContext; +use crate::parser::{DeferredInline, DeferredInlineFlavor}; /// Check if current token consists only of ASCII spaces and/or tabs. /// @@ -666,9 +667,24 @@ fn consume_blank_line(p: &mut MarkdownParser) { pub(crate) fn parse_paragraph(p: &mut MarkdownParser) -> ParsedSyntax { let m = p.start(); + let event_start = p.context().events().len(); + let context = p.inline_container_context(); + let definitions_len = p.link_reference_definitions_len(); let inline_start: usize = p.cur_range().start().into(); parse_inline_item_list(p); let inline_end: usize = p.cur_range().start().into(); + let event_end = p.context().events().len(); + + p.record_deferred_inline(DeferredInline { + event_range: event_start..event_end, + source_range: TextRange::new( + TextSize::from(inline_start as u32), + TextSize::from(inline_end as u32), + ), + flavor: DeferredInlineFlavor::Paragraph, + context, + definitions_len, + }); let has_inline_content = inline_has_non_whitespace(p, inline_start, inline_end); let allow_setext = has_inline_content && allow_setext_heading(p); diff --git a/crates/biome_markdown_parser/src/to_html.rs b/crates/biome_markdown_parser/src/to_html.rs index 96bb4e475b96..b5752e9ca210 100644 --- a/crates/biome_markdown_parser/src/to_html.rs +++ b/crates/biome_markdown_parser/src/to_html.rs @@ -2110,6 +2110,18 @@ mod tests { assert_eq!(html, "

italic and bold

\n"); } + #[test] + fn test_forward_reference_scopes_emphasis() { + assert_eq!( + render("*[foo*][ref]\n\n[ref]: /uri\n"), + "

*foo*

\n" + ); + assert_eq!( + render("[foo *bar][ref]*\n\n[ref]: /uri\n"), + "

foo *bar*

\n" + ); + } + #[test] fn test_emphasis_complex_cases() { // Test: Nested diff --git a/crates/biome_markdown_parser/src/token_source.rs b/crates/biome_markdown_parser/src/token_source.rs index 2247e5a8a58c..e114f4de8829 100644 --- a/crates/biome_markdown_parser/src/token_source.rs +++ b/crates/biome_markdown_parser/src/token_source.rs @@ -49,6 +49,16 @@ impl<'source> MarkdownTokenSource<'source> { pub fn from_str(source: &'source str) -> Self { let lexer = MarkdownLexer::from_str(source); + Self::from_lexer(lexer) + } + + pub fn from_range(source: &'source str, range: TextRange) -> Option { + let lexer = MarkdownLexer::from_range(source, range)?; + + Some(Self::from_lexer(lexer)) + } + + fn from_lexer(lexer: MarkdownLexer<'source>) -> Self { let buffered = BufferedLexer::new(lexer); let mut source = MarkdownTokenSource::new(buffered); @@ -97,8 +107,9 @@ impl<'source> MarkdownTokenSource<'source> { pub fn source_after_current(&self) -> &str { let range = self.lexer.current_range(); let start: usize = range.start().into(); + let end = self.lexer.lexer().range_end(); let source = self.lexer.source(); - &source[start..] + &source[start..end] } /// Returns the full source text. diff --git a/crates/biome_markdown_parser/tests/cst_invariants.rs b/crates/biome_markdown_parser/tests/cst_invariants.rs index 48e176c21a3e..278a49ec8176 100644 --- a/crates/biome_markdown_parser/tests/cst_invariants.rs +++ b/crates/biome_markdown_parser/tests/cst_invariants.rs @@ -1,6 +1,7 @@ use biome_markdown_parser::parse_markdown; use biome_markdown_syntax::{ MarkdownSyntaxKind, MdContinuationIndent, MdListMarkerPrefix, MdOrderedListItem, + MdReferenceLink, }; use biome_rowan::{AstNode, AstNodeList, Direction, TextRange}; @@ -347,3 +348,43 @@ fn no_fixture_has_bullet_list_newline_siblings() { offenders.join("\n") ); } + +#[test] +fn deferred_references_preserve_cst_and_container_metadata() { + let input = "> - *[foo*][ref]\n>\n> [ref]: /uri\n"; + let parsed = parse_markdown(input); + + assert_eq!(parsed.syntax().to_string(), input); + assert!(parsed.diagnostics().is_empty()); + assert!( + parsed + .syntax() + .descendants() + .any(|node| MdReferenceLink::can_cast(node.kind())) + ); + assert!(!parsed.list_tightness().is_empty()); + assert!(!parsed.list_item_indents().is_empty()); + assert!(!parsed.quote_indents().is_empty()); +} + +#[test] +fn deferred_references_preserve_headings_and_diagnostics() { + let heading = "# *[foo*][ref]\n\n[ref]: /uri\n"; + let parsed = parse_markdown(heading); + + assert_eq!(parsed.syntax().to_string(), heading); + assert!( + parsed + .syntax() + .descendants() + .any(|node| MdReferenceLink::can_cast(node.kind())) + ); + + let invalid = format!("{} [ref]\n\n[ref]: /uri\n", ">".repeat(101)); + let parsed = parse_markdown(&invalid); + assert_eq!(parsed.syntax().to_string(), invalid); + assert!(!parsed.diagnostics().is_empty()); + assert!(parsed.diagnostics().windows(2).all(|diagnostics| { + diagnostics[0].span().map(TextRange::start) <= diagnostics[1].span().map(TextRange::start) + })); +} diff --git a/crates/biome_parser/src/diagnostic.rs b/crates/biome_parser/src/diagnostic.rs index 311184fd2855..03489f2d608f 100644 --- a/crates/biome_parser/src/diagnostic.rs +++ b/crates/biome_parser/src/diagnostic.rs @@ -156,6 +156,10 @@ impl ParseDiagnostic { } } + pub fn span(&self) -> Option { + self.span + } + /// Updates the location of this diagnostic and its advices pub fn set_location_offset(&mut self, offset: TextSize) { self.advice_offset = Some(offset); From 10e41a49f53d5dc29ce71e8a7f3dffdbd8cd1e84 Mon Sep 17 00:00:00 2001 From: Emanuele Stoppa Date: Thu, 6 Aug 2026 16:53:06 +0100 Subject: [PATCH 2/2] address regression --- crates/biome_markdown_parser/src/parser.rs | 88 +++++++++++++++------- 1 file changed, 60 insertions(+), 28 deletions(-) diff --git a/crates/biome_markdown_parser/src/parser.rs b/crates/biome_markdown_parser/src/parser.rs index b76f211d1deb..9e405f4bc702 100644 --- a/crates/biome_markdown_parser/src/parser.rs +++ b/crates/biome_markdown_parser/src/parser.rs @@ -2,9 +2,9 @@ use biome_markdown_syntax::MarkdownSyntaxKind; use biome_parser::ParserContext; use biome_parser::event::Event; use biome_parser::prelude::*; -use biome_parser::token_source::Trivia; +use biome_parser::token_source::{BumpWithContext, Trivia}; use biome_parser::{ParserContextCheckpoint, diagnostic::merge_diagnostics}; -use biome_rowan::{TextRange, TextSize}; +use biome_rowan::{TextRange, TextSize, TriviaPieceKind}; use std::cell::Cell; use std::collections::HashMap; use std::hash::{DefaultHasher, Hash, Hasher}; @@ -14,6 +14,7 @@ use crate::lexer::{MarkdownLexContext, MarkdownReLexContext}; use crate::syntax::TAB_STOP_SPACES; use crate::syntax::inline::EmphasisContext; use crate::syntax::parse_error::DEFAULT_MAX_NESTING_DEPTH; +use crate::syntax::reference::normalize_reference_label; use crate::token_source::{MarkdownTokenSource, MarkdownTokenSourceCheckpoint}; /// Options for configuring the markdown parser. @@ -200,6 +201,8 @@ pub(crate) struct InlineContainerContext { pub(crate) struct LinkReferenceDefinitions { /// Source ranges of definition labels in document order. ranges: Vec, + /// Normalized-label hashes corresponding to `ranges`. + hashes: Vec, /// Normalized-label hash to indices in `ranges`. /// /// Each hash can identify multiple ranges because duplicate definitions and @@ -212,6 +215,7 @@ impl LinkReferenceDefinitions { fn insert(&mut self, range: TextRange, hash: u64) { let index = self.ranges.len(); self.ranges.push(range); + self.hashes.push(hash); self.ranges_by_hash.entry(hash).or_default().push(index); } @@ -220,7 +224,7 @@ impl LinkReferenceDefinitions { } fn contains(&self, source: &str, normalized_label: &str) -> bool { - let hash = normalized_label_hash(normalized_label); + let hash = hash_normalized_label(normalized_label); self.ranges_by_hash .get(&hash) .into_iter() @@ -232,18 +236,38 @@ impl LinkReferenceDefinitions { }) } - fn truncate(&mut self, len: usize, hashes: impl IntoIterator) { - self.ranges.truncate(len); - self.ranges_by_hash.clear(); - for (index, hash) in hashes { - self.ranges_by_hash.entry(hash).or_default().push(index); + fn truncate(&mut self, len: usize) { + while self.ranges.len() > len { + let index = self.ranges.len() - 1; + self.ranges.pop(); + let Some(hash) = self.hashes.pop() else { + self.ranges.truncate(len); + self.ranges_by_hash.clear(); + return; + }; + let Some(indices) = self.ranges_by_hash.get_mut(&hash) else { + continue; + }; + if indices.last() == Some(&index) { + indices.pop(); + } else { + indices.retain(|candidate| *candidate != index); + } + let remove_bucket = indices.is_empty(); + if remove_bucket { + self.ranges_by_hash.remove(&hash); + } } } } fn normalized_label_hash(label: &str) -> u64 { + hash_normalized_label(&normalize_reference_label(label)) +} + +fn hash_normalized_label(label: &str) -> u64 { let mut hasher = DefaultHasher::new(); - crate::syntax::reference::normalize_reference_label(label).hash(&mut hasher); + label.hash(&mut hasher); hasher.finish() } @@ -737,8 +761,6 @@ impl<'source> MarkdownParser<'source> { /// but not appear as explicit CST nodes. The token is removed from the /// event stream and attached as `Whitespace` trivia on the next real token. pub fn consume_as_whitespace_trivia(&mut self) { - use biome_parser::token_source::BumpWithContext; - use biome_rowan::TriviaPieceKind; self.source_mut().skip_as_trivia_of_kind_with_context( TriviaPieceKind::Whitespace, MarkdownLexContext::Regular, @@ -816,25 +838,9 @@ impl<'source> MarkdownParser<'source> { self.context.rewind(context); self.source.rewind(source); self.state.deferred_inlines.truncate(deferred_inlines_len); - let hashes = self - .state - .link_reference_definitions - .ranges - .iter() - .take(link_reference_definitions_len) - .enumerate() - .filter_map(|range| { - let (index, range) = range; - self.source - .source_text() - .get(usize::from(range.start())..usize::from(range.end())) - .map(|label| (index, label)) - }) - .map(|(index, label)| (index, normalized_label_hash(label))) - .collect::>(); self.state .link_reference_definitions - .truncate(link_reference_definitions_len, hashes); + .truncate(link_reference_definitions_len); } /// Execute a lookahead operation without consuming tokens. @@ -943,3 +949,29 @@ pub struct MarkdownParserCheckpoint { deferred_inlines_len: usize, link_reference_definitions_len: usize, } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rewind_removes_only_speculative_link_definitions() { + let source = "foo Foo bar"; + let mut parser = MarkdownParser::new(source, MarkdownParserOptions::default()); + parser.record_link_reference_definition(TextRange::new(0.into(), 3.into())); + let checkpoint = parser.checkpoint(); + + parser.record_link_reference_definition(TextRange::new(4.into(), 7.into())); + parser.record_link_reference_definition(TextRange::new(8.into(), 11.into())); + parser.rewind(checkpoint); + + assert!(parser.has_link_reference_definition("foo")); + assert!(!parser.has_link_reference_definition("bar")); + assert_eq!(parser.state.link_reference_definitions.ranges.len(), 1); + assert_eq!(parser.state.link_reference_definitions.hashes.len(), 1); + assert_eq!( + parser.state.link_reference_definitions.ranges_by_hash.len(), + 1 + ); + } +}