refactor(md/parse): two-phase parse

crates/biome_markdown_formatter/tests/specs/markdown/reference_link.md

@@ -14,6 +14,10 @@ text with [inline][ref] in the middle
 
 [link with **bold**][ref]
 
+# *[heading*][ref]
+
+> - [nested][ref]
+
 [world]: https://example.com "World"
 [  label  ]: https://example.com "Label"
 [*foo* bar]: https://example.com "Foo Bar"

crates/biome_markdown_formatter/tests/specs/markdown/reference_link.md.snap

@@ -22,6 +22,10 @@ text with [inline][ref] in the middle
 
 [link with **bold**][ref]
 
+# *[heading*][ref]
+
+> - [nested][ref]
+
 [world]: https://example.com "World"
 [  label  ]: https://example.com "Label"
 [*foo* bar]: https://example.com "Foo Bar"
@@ -51,6 +55,10 @@ text with [inline][ref] in the middle
 
 [link with **bold**][ref]
 
+# *[heading*][ref]
+
+> - [nested][ref]
+
 [world]: https://example.com "World"
 [ label ]: https://example.com "Label"
 [*foo* bar]: https://example.com "Foo Bar"

crates/biome_markdown_parser/src/inline_phase.rs

@@ -0,0 +1,179 @@
+//! Completes inline syntax after Markdown block parsing.
+//!
+//! CommonMark separates parsing into a block phase followed by an inline phase
+//! because link reference definitions are document-global and are not all known
+//! until block parsing finishes. See the
+//! [CommonMark parsing strategy](https://spec.commonmark.org/0.31.2/#appendix-a-parsing-strategy).
+//!
+//! The block parser records each inline event subtree together with its source
+//! range and the number of definitions known at that point. Inline subtrees that
+//! precede later definitions are parsed again with the complete definition index.
+//! Their replacement events, trivia, and diagnostics are merged before the
+//! lossless tree sink constructs the CST, so source text is never rewritten.
+
+use biome_markdown_syntax::MarkdownSyntaxKind::MD_ROOT;
+use biome_parser::Parser;
+use biome_parser::diagnostic::merge_diagnostics;
+use biome_parser::event::Event;
+use biome_parser::token_source::Trivia;
+use biome_rowan::{TextRange, TextSize};
+
+use crate::MarkdownParserOptions;
+use crate::parser::{
+    DeferredInline, DeferredInlineFlavor, LinkReferenceDefinitions, MarkdownParser,
+    MarkdownParserOutput,
+};
+use crate::syntax::header::parse_header_content;
+use crate::syntax::parse_inline_item_list;
+
+/// Resolves deferred inline subtrees and returns an event stream ready for the
+/// lossless tree sink.
+///
+/// The returned output contains no deferred inline records. Event replacement
+/// requires the recorded event ranges to be ordered and non-overlapping.
+pub(crate) fn parse_deferred_inlines(
+    source: &str,
+    options: &MarkdownParserOptions,
+    mut output: MarkdownParserOutput,
+) -> MarkdownParserOutput {
+    debug_assert!(output.events.iter().all(|event| {
+        !matches!(
+            event,
+            Event::Start {
+                forward_parent: Some(_),
+                ..
+            }
+        )
+    }));
+
+    let mut replacements = Vec::new();
+    let mut previous_event_end = 0;
+
+    for deferred in &output.deferred_inlines {
+        if deferred.definitions_len == output.link_reference_definitions.len() {
+            continue;
+        }
+        if deferred.event_range.start < previous_event_end
+            || deferred.event_range.start > deferred.event_range.end
+            || deferred.event_range.end > output.events.len()
+        {
+            continue;
+        }
+
+        let Some(fragment) = parse_inline_fragment(
+            source,
+            deferred,
+            options.clone(),
+            &output.link_reference_definitions,
+        ) else {
+            continue;
+        };
+
+        replacements.push((deferred, fragment));
+        previous_event_end = deferred.event_range.end;
+    }
+
+    let mut events = Vec::with_capacity(output.events.len());
+    let mut old_events = std::mem::take(&mut output.events).into_iter();
+    let mut inline_trivia = Vec::new();
+    let mut inline_diagnostics = Vec::new();
+    let mut reparsed_ranges = Vec::with_capacity(replacements.len());
+    let mut event_position = 0;
+
+    for (deferred, fragment) in replacements {
+        debug_assert!(event_position <= deferred.event_range.start);
+        while event_position < deferred.event_range.start {
+            let Some(event) = old_events.next() else {
+                break;
+            };
+            events.push(event);
+            event_position += 1;
+        }
+        while event_position < deferred.event_range.end {
+            if old_events.next().is_none() {
+                break;
+            }
+            event_position += 1;
+        }
+        events.extend(fragment.events);
+
+        reparsed_ranges.push(deferred.source_range);
+        inline_trivia.extend(fragment.trivia);
+        inline_diagnostics.extend(fragment.diagnostics);
+    }
+    events.extend(old_events);
+    output.events = events;
+
+    let mut range_index = 0;
+    output.trivia.retain(|trivia| {
+        !overlaps_ordered_ranges(trivia.text_range(), &reparsed_ranges, &mut range_index)
+    });
+    output.trivia.extend(inline_trivia);
+    output.trivia.sort_by_key(Trivia::offset);
+
+    range_index = 0;
+    output.diagnostics.retain(|diagnostic| {
+        diagnostic
+            .span()
+            .is_none_or(|span| !overlaps_ordered_ranges(span, &reparsed_ranges, &mut range_index))
+    });
+    output.diagnostics = merge_diagnostics(output.diagnostics, inline_diagnostics);
+    output.deferred_inlines.clear();
+
+    output
+}
+
+fn parse_inline_fragment<'source>(
+    source: &'source str,
+    deferred: &DeferredInline,
+    options: MarkdownParserOptions,
+    definitions: &'source LinkReferenceDefinitions,
+) -> Option<MarkdownParserOutput> {
+    let source_end = TextSize::try_from(source.len()).ok()?;
+    let mut parser = MarkdownParser::new_range(
+        source,
+        TextRange::new(deferred.source_range.start(), source_end),
+        options,
+        deferred.context,
+        definitions,
+    )?;
+
+    let wrapper = parser.start();
+
+    match deferred.flavor {
+        DeferredInlineFlavor::Paragraph => parse_inline_item_list(&mut parser),
+        DeferredInlineFlavor::AtxParagraph => parse_header_content(&mut parser),
+    }
+
+    debug_assert_eq!(
+        parser.cur_range().start(),
+        deferred.source_range.end(),
+        "inline fragment {:?} stopped at {:?} ({:?})",
+        deferred.source_range,
+        parser.cur_range(),
+        (parser.cur(), &source[deferred.source_range])
+    );
+    wrapper.complete(&mut parser, MD_ROOT);
+    let mut output = parser.finish();
+    output.events.remove(0);
+    output.events.pop();
+    Some(output)
+}
+fn overlaps_ordered_ranges(
+    range: TextRange,
+    ranges: &[TextRange],
+    range_index: &mut usize,
+) -> bool {
+    while ranges
+        .get(*range_index)
+        .is_some_and(|candidate| candidate.end() <= range.start())
+    {
+        *range_index += 1;
+    }
+
+    ranges.get(*range_index).is_some_and(|candidate| {
+        candidate
+            .intersect(range)
+            .is_some_and(|intersection| !intersection.is_empty())
+    })
+}

crates/biome_markdown_parser/src/lexer/mod.rs

@@ -9,7 +9,7 @@ use biome_parser::diagnostic::ParseDiagnostic;
 use biome_parser::lexer::{
     LexContext, Lexer, LexerCheckpoint, LexerWithCheckpoint, ReLexer, TokenFlags,
 };
-use biome_rowan::{SyntaxKind, TextSize};
+use biome_rowan::{SyntaxKind, TextRange, TextSize};
 use biome_unicode_table::Dispatch::{self, AMP, *};
 use biome_unicode_table::{is_unicode_punctuation, lookup_byte};
 
@@ -90,6 +90,9 @@ pub(crate) struct MarkdownLexer<'src> {
     /// Source text
     source: &'src str,
 
+    /// Exclusive end of the source range visible to this lexer.
+    end: usize,
+
     /// The start byte position in the source text of the next token.
     position: usize,
 
@@ -210,6 +213,15 @@ impl<'src> Lexer<'src> for MarkdownLexer<'src> {
         self.position
     }
 
+    fn is_eof(&self) -> bool {
+        self.position >= self.end
+    }
+
+    fn byte_at(&self, offset: usize) -> Option<u8> {
+        let position = self.position.checked_add(offset)?;
+        (position < self.end).then(|| self.source.as_bytes()[position])
+    }
+
     fn push_diagnostic(&mut self, diagnostic: ParseDiagnostic) {
         self.diagnostics.push(diagnostic);
     }
@@ -230,15 +242,31 @@ impl<'src> Lexer<'src> for MarkdownLexer<'src> {
 impl<'src> MarkdownLexer<'src> {
     /// Make a new lexer from a str, this is safe because strs are valid utf8
     pub fn from_str(source: &'src str) -> Self {
+        Self::from_valid_range(source, TextSize::from(0), source.len())
+    }
+
+    pub fn from_range(source: &'src str, range: TextRange) -> Option<Self> {
+        let start: usize = range.start().into();
+        let end: usize = range.end().into();
+        source.get(start..end)?;
+
+        Some(Self::from_valid_range(source, range.start(), end))
+    }
+
+    fn from_valid_range(source: &'src str, start: TextSize, end: usize) -> Self {
         Self {
             source,
-            // Start of document is treated as start of line for indentation purposes
-            after_newline: true,
+            end,
+            after_newline: start == TextSize::from(0)
+                || matches!(
+                    source.as_bytes().get(usize::from(start) - 1),
+                    Some(b'\n' | b'\r')
+                ),
             unicode_bom_length: 0,
             current_kind: TOMBSTONE,
-            current_start: TextSize::from(0),
+            current_start: start,
             current_flags: TokenFlags::empty(),
-            position: 0,
+            position: start.into(),
             diagnostics: vec![],
             force_ordered_list_marker: false,
             relex_span: None,
@@ -249,6 +277,10 @@ impl<'src> MarkdownLexer<'src> {
         self.force_ordered_list_marker = value;
     }
 
+    pub fn range_end(&self) -> usize {
+        self.end
+    }
+
     /// Sets the target for the next [MarkdownReLexContext::Span] re-lex.
     pub fn set_relex_span(&mut self, end: usize, kind: MarkdownSyntaxKind) {
         self.relex_span = Some((end, kind));
@@ -589,10 +621,8 @@ impl<'src> MarkdownLexer<'src> {
     /// Returns the byte at position `self.position + offset` or `None` if it is out of bounds.
     #[inline]
     fn byte_at(&self, offset: usize) -> Option<u8> {
-        self.source()
-            .as_bytes()
-            .get(self.position() + offset)
-            .copied()
+        let position = self.position.checked_add(offset)?;
+        (position < self.end).then(|| self.source.as_bytes()[position])
     }
 
     /// Peeks at the next byte
@@ -1221,7 +1251,7 @@ impl<'src> MarkdownLexer<'src> {
     /// Returns `true` if the parser is at or passed the end of the file.
     #[inline]
     fn is_eof(&self) -> bool {
-        self.position >= self.source.len()
+        self.position >= self.end
     }
 
     /// Consume textual characters until hitting special markdown syntax.
@@ -1385,15 +1415,15 @@ impl<'src> MarkdownLexer<'src> {
     }
 
     fn is_ordered_list_marker_at(&self, idx: usize) -> bool {
-        if idx >= self.source.len() {
+        if idx >= self.end {
             return false;
         }
 
         let bytes = self.source.as_bytes();
         let mut pos = idx;
         let mut digit_count = 0;
 
-        while pos < bytes.len() && bytes[pos].is_ascii_digit() {
+        while pos < self.end && bytes[pos].is_ascii_digit() {
             digit_count += 1;
             if digit_count > MAX_ORDERED_LIST_MARKER_DIGITS {
                 return false;
@@ -1405,12 +1435,12 @@ impl<'src> MarkdownLexer<'src> {
             return false;
         }
 
-        if pos >= bytes.len() || !(bytes[pos] == b'.' || bytes[pos] == b')') {
+        if pos >= self.end || !(bytes[pos] == b'.' || bytes[pos] == b')') {
             return false;
         }
         pos += 1;
 
-        if pos >= bytes.len() {
+        if pos >= self.end {
             return true;
         }
 
@@ -1423,7 +1453,7 @@ impl<'src> MarkdownLexer<'src> {
     fn is_at_trailing_hash_closing_whitespace(&self) -> bool {
         let mut i = self.position;
         let bytes = self.source.as_bytes();
-        let len = bytes.len();
+        let len = self.end;
 
         let mut saw_ws = false;
         while i < len {
@@ -1526,7 +1556,7 @@ impl<'src> MarkdownLexer<'src> {
         while let Some(b'_') = self.byte_at(offset) {
             offset += 1;
         }
-        let after = self.source[self.position + offset..].chars().next();
+        let after = self.source[self.position + offset..self.end].chars().next();
         is_word_char(after)
     }
 
@@ -1626,7 +1656,7 @@ impl<'src> ReLexer<'src> for MarkdownLexer<'src> {
                     .relex_span
                     .take()
                     .expect("set_relex_span must be called before a Span re-lex");
-                debug_assert!(end > self.position && end <= self.source.len());
+                debug_assert!(end > self.position && end <= self.end);
                 self.position = end;
                 kind
             }