From 9853fac38999326774651bb98dd897c398fbc659 Mon Sep 17 00:00:00 2001 From: James Yang Date: Thu, 13 Aug 2026 14:00:29 -0400 Subject: [PATCH] Add to_markdown_pages for PDF per-page extraction. Expose a thin wrapper over pdf-inspector's page API so callers get page boundaries and OCR flags without forcing PDFs through the document model (Closes #62). --- README.md | 6 +++- src/formats/mod.rs | 5 +-- src/formats/pdf.rs | 85 ++++++++++++++++++++++++++++++++++++++++++++-- src/lib.rs | 32 +++++++++++++++-- 4 files changed, 120 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index ed4cb9e..b9b4238 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,9 @@ let markdown = anydoc::to_markdown_bytes(&bytes, anydoc::Format::Csv)?; // Or stop at the document model, which also carries embedded assets: let document = anydoc::to_document(&bytes, None)?; + +// PDFs can also return per-page Markdown (page boundaries + OCR flags): +let pages = anydoc::to_markdown_pages("report.pdf")?; ``` ## Features @@ -133,7 +136,7 @@ let document = anydoc::to_document(&bytes, None)?; - **Content-based format detection.** The format is read from the bytes themselves (PDF header, RTF open group, OLE stream names, ZIP package mimetype), so mislabeled files still convert correctly. - **Fast.** Pure Rust, no ML models, no external services. Median conversion time is under 5ms per document. - **Bindings that stay out of the way.** Node.js conversion runs on the libuv thread pool and never blocks the event loop; Python releases the GIL so other threads keep running. TypeScript types and Python stubs ship with the packages. -- **PDF support built in.** Text-based PDFs convert locally through [pdf-inspector](https://github.com/firecrawl/pdf-inspector), no OCR service required. +- **PDF support built in.** Text-based PDFs convert locally through [pdf-inspector](https://github.com/firecrawl/pdf-inspector), no OCR service required. Use `to_markdown` / `to_markdown_bytes` for one Markdown blob, or `to_markdown_pages` / `to_markdown_pages_bytes` when you need per-page output and OCR flags. - **Agent ready.** Ships as an [Agent Skill](#agent-skill): one `npx skills add firecrawl/anydoc` and any agent can read office documents. ## Supported formats @@ -242,6 +245,7 @@ document bytes │ └─► GFM serializer → Markdown │ └─► PDF → pdf-inspector → Markdown directly + (or per-page via to_markdown_pages) ``` Because every format funnels through the same document model and serializer, output quirks get fixed once. A table-escaping fix for docx is automatically a table-escaping fix for rtf, odt, and everything else. diff --git a/src/formats/mod.rs b/src/formats/mod.rs index d570b26..0880b4a 100644 --- a/src/formats/mod.rs +++ b/src/formats/mod.rs @@ -30,9 +30,10 @@ pub fn parse(bytes: &[u8], format: Format) -> Result { Format::Doc => doc::parse(bytes), Format::Ppt => ppt::parse(bytes), // pdf-inspector produces Markdown directly; there is no document - // model for PDFs. `to_markdown_bytes` routes them to `pdf`. + // model for PDFs. `to_markdown_bytes` / `to_markdown_pages_bytes` + // route them to `pdf`. Format::Pdf => Err(ConvertError::Unsupported( - "PDF converts directly to Markdown; use to_markdown or to_markdown_bytes".to_string(), + "PDF converts directly to Markdown; use to_markdown, to_markdown_bytes, or to_markdown_pages".to_string(), )), } } diff --git a/src/formats/pdf.rs b/src/formats/pdf.rs index 74a36e1..85cf846 100644 --- a/src/formats/pdf.rs +++ b/src/formats/pdf.rs @@ -3,14 +3,33 @@ //! Unlike the other frontends, pdf-inspector emits Markdown itself, so PDFs //! bypass the document model and the shared GFM writer. Scanned and //! image-only PDFs need OCR, which is out of scope here; they error as -//! unsupported. Pages flagged for OCR in an otherwise text-based document -//! degrade with a log, consistent with the crate-wide recovery policy. +//! unsupported from [`to_markdown`]. Per-page extraction via +//! [`to_markdown_pages`] surfaces OCR needs on each page instead, so callers +//! can route those pages elsewhere. Pages flagged for OCR in an otherwise +//! text-based document degrade with a log from [`to_markdown`], consistent +//! with the crate-wide recovery policy. //! //! [pdf-inspector]: https://github.com/firecrawl/pdf-inspector use crate::error::ConvertError; use pdf_inspector::PdfError; +/// One PDF page converted to Markdown. +/// +/// `page` is 0-indexed. When [`Self::needs_ocr`] is true, [`Self::markdown`] +/// is empty and [`Self::ocr_reason`] may name why extraction was unreliable. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MarkdownPage { + /// 0-indexed page number. + pub page: u32, + /// Formatted Markdown for this page, or empty when OCR is required. + pub markdown: String, + /// True when text on this page is unreliable or missing. + pub needs_ocr: bool, + /// Machine-readable OCR reason when the cause is known. + pub ocr_reason: Option, +} + pub fn to_markdown(bytes: &[u8]) -> Result { let result = pdf_inspector::process_pdf_mem(bytes).map_err(map_error)?; if !result.pages_needing_ocr.is_empty() { @@ -37,6 +56,33 @@ pub fn to_markdown(bytes: &[u8]) -> Result { } } +/// Extract Markdown for each page of a PDF. +/// +/// Unlike [`to_markdown`], this never collapses the document into one string +/// and does not fail solely because some (or all) pages need OCR. Each page +/// carries its own Markdown and OCR flag so callers can mix direct extraction +/// with an external OCR path. +pub fn to_markdown_pages(bytes: &[u8]) -> Result, ConvertError> { + let result = pdf_inspector::extract_pages_markdown_mem(bytes, None).map_err(map_error)?; + if !result.pages_needing_ocr.is_empty() { + log::warn!( + "{} of {} pages need OCR and were not extracted", + result.pages_needing_ocr.len(), + result.pages.len() + ); + } + Ok(result + .pages + .into_iter() + .map(|page| MarkdownPage { + page: page.page, + markdown: page.markdown, + needs_ocr: page.needs_ocr, + ocr_reason: page.ocr_reason, + }) + .collect()) +} + fn map_error(e: PdfError) -> ConvertError { match e { PdfError::Encrypted => ConvertError::Encrypted, @@ -46,3 +92,38 @@ fn map_error(e: PdfError) -> ConvertError { PdfError::Parse(detail) => ConvertError::malformed(detail), } } + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + + fn text_pdf() -> Vec { + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests") + .join("fixtures") + .join("pdf") + .join("text.pdf"); + std::fs::read(path).expect("pdf fixture") + } + + #[test] + fn to_markdown_pages_returns_at_least_one_page() { + let pages = to_markdown_pages(&text_pdf()).expect("extract pages"); + assert!(!pages.is_empty(), "expected at least one page"); + for (i, page) in pages.iter().enumerate() { + assert_eq!(page.page, i as u32); + assert!( + page.needs_ocr || !page.markdown.trim().is_empty(), + "page {} should have markdown or need OCR", + page.page + ); + } + } + + #[test] + fn to_markdown_still_returns_a_single_blob() { + let markdown = to_markdown(&text_pdf()).expect("extract markdown"); + assert!(!markdown.trim().is_empty()); + } +} diff --git a/src/lib.rs b/src/lib.rs index efba6ff..41893e4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,6 +15,7 @@ mod render; mod shared; pub use error::ConvertError; +pub use formats::pdf::MarkdownPage; use render::markdown::document_to_markdown; @@ -32,8 +33,10 @@ pub enum Format { /// OpenDocument Text (`.odt`). Odt, /// Converted with [pdf-inspector], which emits Markdown directly: - /// [`to_document`] is unsupported for PDFs. Scanned/image-only PDFs - /// (needing OCR) error as unsupported. + /// [`to_document`] is unsupported for PDFs. Use [`to_markdown`], + /// [`to_markdown_bytes`], or [`to_markdown_pages`]. Scanned/image-only + /// PDFs (needing OCR) error as unsupported from the single-blob APIs; + /// [`to_markdown_pages`] reports OCR needs per page instead. /// /// [pdf-inspector]: https://github.com/firecrawl/pdf-inspector Pdf, @@ -125,11 +128,34 @@ pub fn to_markdown_bytes( Ok(document_to_markdown(&to_document(bytes, format)?)) } +/// Convert a PDF file to per-page Markdown. +/// +/// This is the page-aware counterpart of [`to_markdown`] for PDFs. Other +/// formats should keep using [`to_markdown`] or [`to_document`]. See +/// [`to_markdown_pages_bytes`] for the in-memory form and page semantics. +pub fn to_markdown_pages(path: impl AsRef) -> Result, ConvertError> { + let bytes = std::fs::read(path.as_ref())?; + to_markdown_pages_bytes(&bytes) +} + +/// Convert an in-memory PDF to per-page Markdown. +/// +/// Wraps pdf-inspector's per-page extraction. Unlike [`to_markdown_bytes`], +/// the result keeps page boundaries and reports OCR needs on each +/// [`MarkdownPage`] instead of failing when the whole document needs OCR. +/// +/// `page` is 0-indexed. Pages that need OCR have empty `markdown` and +/// `needs_ocr = true`. +pub fn to_markdown_pages_bytes(bytes: &[u8]) -> Result, ConvertError> { + formats::pdf::to_markdown_pages(bytes) +} + /// Parse an in-memory document into the document model. Pass a [`Format`] to /// select the parser, or `None` to detect it from the content. /// /// Unsupported for [`Format::Pdf`]: PDF conversion produces Markdown -/// directly and has no document-model form; use [`to_markdown_bytes`]. +/// directly and has no document-model form; use [`to_markdown_bytes`] or +/// [`to_markdown_pages_bytes`]. pub fn to_document( bytes: &[u8], format: impl Into>,