diff --git a/docs/en/api/02-resources.md b/docs/en/api/02-resources.md index 4319fe6c76..4c2af55b43 100644 --- a/docs/en/api/02-resources.md +++ b/docs/en/api/02-resources.md @@ -12,19 +12,19 @@ OpenViking supports various resource types, categorized by functionality: | Type | Extensions | Description | |------|------------|-------------| -| PDF | `.pdf` | Supports local parsing and MinerU API conversion | +| PDF | `.pdf` | Uses pdf-inspector for page text, headings, tables, and reading order; pdfplumber extracts images after each page's text. Pages that require OCR are kept with explicit warnings | | Markdown | `.md`, `.markdown`, `.mdown`, `.mkd` | Native support, extracts structure and stores in segments | | HTML | `.html`, `.htm` | Cleans navigation/ads and extracts content, converts to Markdown | -| Word | `.docx` | Extracts text, headings, tables and converts to Markdown | +| Word | `.doc`, `.docx` | Uses AnyDoc to preserve text, headings, tables, and embedded image positions in Markdown | | Plain Text | `.txt`, `.text` | Direct import and processing | -| EPUB | `.epub` | E-book format, supports ebooklib or manual extraction | +| EPUB | `.epub` | Uses AnyDoc to convert the e-book structure and embedded images to Markdown | **Spreadsheets & Presentations** | Type | Extensions | Description | |------|------------|-------------| -| Excel | `.xlsx`, `.xls`, `.xlsm` | Supports new and legacy Excel formats, converts to Markdown tables by worksheet | -| PowerPoint | `.pptx` | Extracts content by slide, supports extracting notes | +| Excel | `.xlsx`, `.xls`, `.xlsm` | Uses AnyDoc to convert worksheets to Markdown | +| PowerPoint | `.pptx` | Uses AnyDoc to preserve slide content and embedded image positions; speaker notes stay in the same resource under `Speaker Notes` sections | **Code** diff --git a/docs/en/configuration/01-server.md b/docs/en/configuration/01-server.md index a137b97fa8..75b373107d 100644 --- a/docs/en/configuration/01-server.md +++ b/docs/en/configuration/01-server.md @@ -336,7 +336,6 @@ Parsers live under `parsers`: "audio": {}, "video": {}, "markdown": {}, - "excel": {}, "html": {}, "text": {}, "directory": {}, @@ -357,8 +356,7 @@ Parsers live under `parsers`: | `code` | Repository file types, ignore rules, and network safety | | `image` | Image understanding and OCR | | `audio`, `video` | Audio/video parsing | -| `markdown`, `html`, `text` | Text document chunking | -| `excel` | Workbook parsing and chunking | +| `markdown`, `html`, `text` | Text document chunking; AnyDoc-backed Office and EPUB files reuse the `markdown` sectioning settings | | `directory` | Directory scanning and ignore rules | | `feishu` | Feishu/Lark access and parsing | | `webfeed` | Sitemap, RSS, and Atom ingestion | diff --git a/docs/zh/api/02-resources.md b/docs/zh/api/02-resources.md index 6eaf34ce1a..980dedca57 100644 --- a/docs/zh/api/02-resources.md +++ b/docs/zh/api/02-resources.md @@ -11,18 +11,18 @@ OpenViking 支持多种资源类型，按照功能分类如下： 文档类 | 类型 | 扩展名 | 说明 | |------|--------|------| -| PDF | `.pdf` | 支持本地解析和 MinerU API 转换 | +| PDF | `.pdf` | 由 pdf-inspector 提取分页文本、标题、表格和阅读顺序，pdfplumber 在每页正文后提取图片；需要 OCR 的页面会保留并给出明确警告 | | Markdown | `.md`, `.markdown`, `.mdown`, `.mkd` | 原生支持，会提取结构并分段存储 | | HTML | `.html`, `.htm` | 清理导航/广告后提取内容，转换为 Markdown | -| Word | `.docx` | 提取文本、标题、表格并转换为 Markdown | +| Word | `.doc`, `.docx` | 通过 AnyDoc 将文本、标题、表格和嵌入图片的原始位置转换为 Markdown | | 纯文本 | `.txt`, `.text` | 直接导入处理 | -| EPUB | `.epub` | 电子书格式，支持 ebooklib 或手动提取 | +| EPUB | `.epub` | 通过 AnyDoc 将电子书结构和嵌入图片转换为 Markdown | 表格类 | 类型 | 扩展名 | 说明 | |------|--------|------| -| Excel | `.xlsx`, `.xls`, `.xlsm` | 支持新版和老版 Excel，按工作表转换为 Markdown 表格 | -| PowerPoint | `.pptx` | 按幻灯片提取内容，支持提取备注 | +| Excel | `.xlsx`, `.xls`, `.xlsm` | 通过 AnyDoc 将工作表转换为 Markdown | +| PowerPoint | `.pptx` | 通过 AnyDoc 保留幻灯片内容和嵌入图片位置；演讲者备注保留在同一资源的 `Speaker Notes` 分节中 | 代码类 | 类型 | 资源名 | 说明 | diff --git a/docs/zh/configuration/01-server.md b/docs/zh/configuration/01-server.md index c4a3192e45..62d76cdd7e 100644 --- a/docs/zh/configuration/01-server.md +++ b/docs/zh/configuration/01-server.md @@ -336,7 +336,6 @@ Provider 和密钥管理配置见[加密指南](../guides/08-encryption.md)。 "audio": {}, "video": {}, "markdown": {}, - "excel": {}, "html": {}, "text": {}, "directory": {}, @@ -357,8 +356,7 @@ Provider 和密钥管理配置见[加密指南](../guides/08-encryption.md)。 | `code` | 代码仓库文件类型、忽略规则和安全限制 | | `image` | 图片理解和 OCR | | `audio`、`video` | 音视频内容解析 | -| `markdown`、`html`、`text` | 文本文档分段 | -| `excel` | Excel 工作表解析与分段 | +| `markdown`、`html`、`text` | 文本文档分段；由 AnyDoc 解析的 Office 和 EPUB 文件复用 `markdown` 分段配置 | | `directory` | 目录扫描和忽略规则 | | `feishu` | 飞书文档访问与解析 | | `webfeed` | Sitemap、RSS 和 Atom 导入 | diff --git a/examples/ov.conf.example b/examples/ov.conf.example index 9056a72865..422869a414 100644 --- a/examples/ov.conf.example +++ b/examples/ov.conf.example @@ -223,14 +223,10 @@ }, "parsers": { "pdf": { - "strategy": "auto", "max_content_length": 100000, "max_section_size": 4000, "section_size_flexibility": 0.3, "max_section_chars": 6000, - "mineru_endpoint": "https://mineru.example.com/api/v1", - "mineru_api_key": "{your-mineru-api-key}", - "mineru_timeout": 300.0, }, "code": { "github_raw_domain": "raw.githubusercontent.com", diff --git a/openviking/parse/image_validation.py b/openviking/parse/image_validation.py new file mode 100644 index 0000000000..f8ebe4b33c --- /dev/null +++ b/openviking/parse/image_validation.py @@ -0,0 +1,58 @@ +# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. +# SPDX-License-Identifier: AGPL-3.0 +"""Shared acceptance policy for images embedded in parsed documents.""" + +import io +from pathlib import Path + +from openviking_cli.utils.logger import get_logger + +logger = get_logger(__name__) + +IMAGE_MIN_SIDE = 14 +IMAGE_MIN_PIXELS = 196 +IMAGE_MAX_PIXELS = 36_000_000 +IMAGE_MIN_ASPECT_RATIO = 1 / 150 +IMAGE_MAX_ASPECT_RATIO = 150 +IMAGE_MAX_FILE_BYTES = 10 * 1024 * 1024 + + +def is_valid_image(image_bytes: bytes, source_path: Path) -> bool: + """Return whether an extracted image is safe and useful to ingest.""" + if len(image_bytes) > IMAGE_MAX_FILE_BYTES: + logger.warning(f"[ImageValidation] Image exceeds 10MB, skipping: {source_path}") + return False + + try: + from PIL import Image + + with Image.open(io.BytesIO(image_bytes)) as image: + width, height = image.size + except Exception as exc: + logger.warning( + f"[ImageValidation] Cannot read image dimensions, skipping {source_path}: {exc}" + ) + return False + + if width <= IMAGE_MIN_SIDE or height <= IMAGE_MIN_SIDE: + logger.warning( + f"[ImageValidation] Image side too small ({width}x{height}), skipping: {source_path}" + ) + return False + + pixels = width * height + if pixels < IMAGE_MIN_PIXELS or pixels > IMAGE_MAX_PIXELS: + logger.warning( + f"[ImageValidation] Image pixel count out of range ({pixels}), skipping: {source_path}" + ) + return False + + aspect_ratio = width / height + if aspect_ratio < IMAGE_MIN_ASPECT_RATIO or aspect_ratio > IMAGE_MAX_ASPECT_RATIO: + logger.warning( + f"[ImageValidation] Image aspect ratio out of range ({aspect_ratio:.4f}), " + f"skipping: {source_path}" + ) + return False + + return True diff --git a/openviking/parse/parsers/__init__.py b/openviking/parse/parsers/__init__.py index 5f17740d03..2bec8f7259 100644 --- a/openviking/parse/parsers/__init__.py +++ b/openviking/parse/parsers/__init__.py @@ -1,26 +1,20 @@ # Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. # SPDX-License-Identifier: AGPL-3.0 +from .anydoc import AnyDocParser from .base_parser import BaseParser -from .epub import EPubParser -from .excel import ExcelParser from .html import HTMLParser from .markdown import MarkdownParser from .pdf import PDFParser -from .powerpoint import PowerPointParser from .text import TextParser -from .word import WordParser from .zip_parser import ZipParser __all__ = [ + "AnyDocParser", "BaseParser", - "EPubParser", - "ExcelParser", "HTMLParser", "MarkdownParser", "PDFParser", - "PowerPointParser", "TextParser", - "WordParser", "ZipParser", ] diff --git a/openviking/parse/parsers/anydoc.py b/openviking/parse/parsers/anydoc.py new file mode 100644 index 0000000000..9542a44254 --- /dev/null +++ b/openviking/parse/parsers/anydoc.py @@ -0,0 +1,808 @@ +# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. +# SPDX-License-Identifier: AGPL-3.0 +"""Unified AnyDoc parser for Office documents and EPUB files.""" + +import asyncio +import html +import re +import time +from dataclasses import dataclass +from importlib.metadata import version +from pathlib import Path +from typing import Any, Iterable, List, Optional, Union + +from openviking.parse.accessors.mime_types import get_preferred_extension +from openviking.parse.base import ParseResult +from openviking.parse.image_validation import is_valid_image +from openviking.parse.parsers.base_parser import BaseParser +from openviking_cli.utils.config.parser_config import ParserConfig +from openviking_cli.utils.logger import get_logger + +logger = get_logger(__name__) + + +@dataclass(frozen=True) +class _RenderedDocument: + markdown: str + detected_format: str + warnings: list[str] + assets_referenced: int + images_extracted: int + + +@dataclass(frozen=True) +class _ResolvedAnchor: + fragment: str + emit_html: bool + + +@dataclass +class _TextRun: + text: str + style: Any + + +class _AnyDocMarkdownRenderer: + """Render AnyDoc's public document model while materializing image assets.""" + + def __init__(self, document: Any, *, source_format: str, resource_name: str, storage: Any): + self.document = document + self.source_format = source_format + self.resource_name = resource_name + self.storage = storage + self.assets = {asset.id: asset for asset in document.assets} + self.asset_paths: dict[int, Optional[str]] = {} + self.warnings: list[str] = [] + self.assets_referenced: set[int] = set() + self.images_extracted = 0 + self.note_numbers = self._number_notes() + self.anchors = self._resolve_anchors() + + def render(self) -> str: + parts = [part for block in self.document.blocks if (part := self._render_block(block))] + + rendered_note_numbers: set[int] = set() + ordered_notes = sorted( + ( + (self.note_numbers[note.id], note) + for note in self.document.notes + if note.id in self.note_numbers + ), + key=lambda item: item[0], + ) + for number, note in ordered_notes: + if number in rendered_note_numbers: + continue + body = self._render_blocks(note.blocks) + if not body: + continue + rendered_note_numbers.add(number) + lines = body.splitlines() + definition = f"[^{number}]: {lines[0]}" + if len(lines) > 1: + definition += "\n" + "\n".join(f" {line}" if line else "" for line in lines[1:]) + parts.append(definition) + + markdown = "\n\n".join(parts) + return f"{markdown}\n" if markdown else "" + + def _render_blocks(self, blocks: Iterable[Any]) -> str: + return "\n\n".join(rendered for block in blocks if (rendered := self._render_block(block))) + + def _render_block(self, block: Any) -> str: + kind = block.kind + if kind == "heading": + text = self._render_inlines(block.content or [], context="heading").strip() + if not text: + return "" + level = min(max(int(block.level or 1), 1), 6) + return f"{'#' * level} {text}" + if kind == "paragraph": + return self._render_inlines(block.content or [], context="block").strip() + if kind == "list": + return self._render_list(block.list) + if kind == "table": + table = block.table + if table.kind == "layout" and self._table_is_single_cell(table): + return self._render_blocks(table.grid[0][0].cell.blocks) + return self._render_table(table) + if kind == "block_quote": + inner = self._render_blocks(block.blocks or []) + if not inner: + return "" + if self.source_format == "pptx": + return f"### Speaker Notes\n\n{inner}" + return "\n".join(">" if not line else f"> {line}" for line in inner.splitlines()) + if kind == "code_block": + body = (block.text or "").rstrip("\n") + fence = self._backtick_fence(body, 3) + return f"{fence}{block.lang or ''}\n{body}\n{fence}" + if kind == "rule": + return "---" + raise RuntimeError(f"Unsupported AnyDoc block kind: {kind}") + + def _render_inlines( + self, inlines: Iterable[Any], *, context: str, in_label: bool = False + ) -> str: + normalized = self._normalize_inlines(inlines) + parts: list[str] = [] + for index, inline in enumerate(normalized): + if isinstance(inline, _TextRun): + next_inline = normalized[index + 1] if index + 1 < len(normalized) else None + trailing_active = ( + isinstance(next_inline, _TextRun) + and self._style_key(next_inline.style) != self._style_key(None) + ) or ( + next_inline is not None + and not isinstance(next_inline, _TextRun) + and next_inline.kind in {"link", "image", "note_ref"} + ) + rendered_so_far = "".join(parts) + parts.append( + self._render_text( + inline.text or "", + inline.style, + context=context, + in_label=in_label, + trailing_active=trailing_active, + at_line_start=not rendered_so_far or rendered_so_far.endswith("\n"), + ) + ) + continue + + kind = inline.kind + if kind == "link": + target = inline.target + if not target.value: + parts.append(self._render_inlines(inline.content or [], context=context)) + continue + label = self._render_inlines(inline.content or [], context=context, in_label=True) + if target.kind == "anchor": + resolved = self.anchors.get(target.value) + if resolved is None: + parts.append(self._render_inlines(inline.content or [], context=context)) + continue + url = f"#{resolved.fragment}" + elif target.kind in {"external", "relative"}: + url = target.value + else: + raise RuntimeError(f"Unsupported AnyDoc link target kind: {target.kind}") + if label.strip(): + parts.append(f"[{label}]({self._format_url(url)})") + elif target.kind != "anchor": + escaped = self._escape_text( + url, context=context, in_label=True, trailing_active=True + ) + parts.append(f"[{escaped}]({self._format_url(url)})") + elif kind == "image": + parts.append(self._render_image(inline, context=context, in_label=in_label)) + elif kind == "anchor": + resolved = self.anchors.get(inline.anchor) + if resolved and resolved.emit_html: + parts.append(f'') + elif kind == "note_ref": + number = self.note_numbers.get(inline.note_id) + if number is not None: + parts.append(f"[^{number}]") + elif kind == "line_break": + parts.append( + "\\\n" if context == "block" else "\n" if context == "table_cell" else " " + ) + else: + raise RuntimeError(f"Unsupported AnyDoc inline kind: {kind}") + return "".join(parts) + + def _normalize_inlines(self, inlines: Iterable[Any]) -> list[Any]: + """Mirror AnyDoc's run normalization before replacing embedded images.""" + normalized: list[Any] = [] + plain_style = self._style_key(None) + for inline in inlines: + if inline.kind == "anchor": + resolved = self.anchors.get(inline.anchor) + if resolved is None or not resolved.emit_html: + continue + if inline.kind != "text": + normalized.append(inline) + continue + if not inline.text: + continue + + style = None if inline.text.isspace() else inline.style + style_key = self._style_key(style) + if isinstance(normalized[-1] if normalized else None, _TextRun): + previous = normalized[-1] + if self._style_key(previous.style) == style_key: + previous.text += inline.text + continue + if ( + style_key != plain_style + and not style_key[3] + and len(normalized) >= 2 + and isinstance(normalized[-1], _TextRun) + and normalized[-1].text.isspace() + and self._style_key(normalized[-1].style) == plain_style + and isinstance(normalized[-2], _TextRun) + and self._style_key(normalized[-2].style) == style_key + ): + whitespace = normalized.pop().text + normalized[-1].text += whitespace + inline.text + continue + normalized.append(_TextRun(text=inline.text, style=style)) + return normalized + + @staticmethod + def _style_key(style: Any) -> tuple[bool, bool, bool, bool]: + if style is None: + return False, False, False, False + return style.bold, style.italic, style.strike, style.code + + def _render_text( + self, + text: str, + style: Any, + *, + context: str, + in_label: bool, + trailing_active: bool, + at_line_start: bool, + ) -> str: + if style is None or not any((style.bold, style.italic, style.strike, style.code)): + return self._escape_text( + text, + context=context, + in_label=in_label, + trailing_active=trailing_active, + at_line_start=at_line_start, + ) + + leading = text[: len(text) - len(text.lstrip())] + trailing = text[len(text.rstrip()) :] + core_end = len(text) - len(trailing) if trailing else len(text) + core = text[len(leading) : core_end] + if not core: + return text + if style.code: + flattened = core.replace("\n", " ") + fence = self._backtick_fence(flattened, 1) + pad = " " if flattened.startswith("`") or flattened.endswith("`") else "" + rendered = f"{fence}{pad}{flattened}{pad}{fence}" + else: + opening = "" + if style.strike: + opening += "~~" + if style.bold: + opening += "**" + if style.italic: + opening += "*" + escaped = self._escape_text( + core, + context=context, + styled=True, + in_label=in_label, + ) + rendered = f"{opening}{escaped}{opening[::-1]}" + return f"{leading}{rendered}{trailing}" + + def _render_image(self, inline: Any, *, context: str, in_label: bool) -> str: + alt = (inline.alt or "").strip() + source = inline.source + escaped_alt = self._escape_text(alt, context=context, in_label=True) + if source.kind == "external": + return f"![{escaped_alt}]({self._format_url(source.url)})" + if source.kind == "unavailable": + self.warnings.append(f"Embedded image is unavailable: {alt or ''}") + return self._escape_text(alt, context=context, in_label=in_label) + if source.kind != "asset": + raise RuntimeError(f"Unsupported AnyDoc image source kind: {source.kind}") + + asset_id = source.asset_id + self.assets_referenced.add(asset_id) + image_ref = self._materialize_asset(asset_id) + if image_ref is None: + return self._escape_text(alt, context=context, in_label=in_label) + return f"![{escaped_alt}]({self._format_url(image_ref)})" + + def _materialize_asset(self, asset_id: int) -> Optional[str]: + if asset_id in self.asset_paths: + return self.asset_paths[asset_id] + asset = self.assets.get(asset_id) + if asset is None: + self.warnings.append(f"AnyDoc image references missing asset {asset_id}") + self.asset_paths[asset_id] = None + return None + if not asset.media_type.lower().startswith("image/"): + self.warnings.append( + f"AnyDoc asset {asset_id} is not an image ({asset.media_type}); kept as alt text" + ) + self.asset_paths[asset_id] = None + return None + + origin_suffix = Path(asset.origin_part).suffix.lower() + extension = origin_suffix or get_preferred_extension(asset.media_type) or ".png" + if not re.fullmatch(r"\.[a-z0-9]{1,10}", extension): + extension = get_preferred_extension(asset.media_type) or ".png" + display_path = Path(f"anydoc_asset_{asset_id}{extension}") + if not is_valid_image(asset.data, display_path): + self.warnings.append( + f"AnyDoc asset {asset_id} is not an ingestable image ({asset.media_type})" + ) + self.asset_paths[asset_id] = None + return None + + try: + path = self.storage.save_image( + self.resource_name, + asset.data, + filename=f"anydoc_asset_{asset_id}", + extension=extension, + ) + relative = path.relative_to(self.storage.media_dir).as_posix() + except Exception as exc: + self.warnings.append(f"Failed to save AnyDoc image asset {asset_id}: {exc}") + self.asset_paths[asset_id] = None + return None + + self.asset_paths[asset_id] = relative + self.images_extracted += 1 + return relative + + def _render_list(self, list_model: Any) -> str: + if list_model is None or not list_model.items: + return "" + rendered_items: list[str] = [] + loose = False + for index, item in enumerate(list_model.items): + ordinal = int(list_model.start) + index + if item.marker_label: + marker = ( + "- " + f"{self._escape_text(item.marker_label, context='block', at_line_start=True)} " + ) + elif list_model.marker == "bullet": + marker = "- " + elif list_model.marker == "decimal": + marker = f"{ordinal}. " + else: + marker = f"- {self._marker_label(list_model.marker, ordinal)} " + checkbox = "[x] " if item.checked is True else "[ ] " if item.checked is False else "" + body = self._render_blocks(item.blocks) + if len(item.blocks) > 1: + loose = True + lines = body.splitlines() or [""] + indent = " " * len(marker) + rendered = f"{marker}{checkbox}{lines[0]}" + for line in lines[1:]: + if not line: + loose = True + rendered += "\n" + else: + rendered += f"\n{indent}{line}" + rendered_items.append(rendered) + return ("\n\n" if loose else "\n").join(rendered_items) + + def _render_table(self, table: Any) -> str: + if table is None or not table.grid: + return "" + width = max((len(row) for row in table.grid), default=0) + rows: list[list[tuple[str, bool]]] = [] + for row in table.grid: + rendered_row: list[tuple[str, bool]] = [] + for slot in row: + if slot.kind == "origin": + rendered_row.append((self._render_cell(slot.cell), False)) + elif slot.kind == "covered": + rendered_row.append(("", True)) + else: + raise RuntimeError(f"Unsupported AnyDoc table slot kind: {slot.kind}") + rendered_row.extend(("", False) for _ in range(width - len(rendered_row))) + rows.append(rendered_row) + while len(rows) > 1 and all(not text and not covered for text, covered in rows[-1]): + rows.pop() + width = max( + ( + max((index + 1 for index, cell in enumerate(row) if cell[0] or cell[1]), default=0) + for row in rows + ), + default=0, + ) + if width == 0: + return "" + rows = [row[:width] for row in rows] + if table.header_rows >= 1: + header = [text for text, _ in rows.pop(0)] + else: + header = [""] * width + lines = [self._format_table_row(header), self._format_table_row(["---"] * width)] + lines.extend(self._format_table_row([text for text, _ in row]) for row in rows) + return "\n".join(lines) + + def _render_cell(self, cell: Any) -> str: + if cell is None: + return "" + parts: list[str] = [] + for block in cell.blocks: + kind = block.kind + if kind == "heading": + text = self._render_inlines(block.content or [], context="table_cell").strip() + if text: + parts.append(f"**{text}**") + elif kind == "paragraph": + text = self._render_inlines(block.content or [], context="table_cell") + if text.strip(): + parts.append(text) + elif kind == "list": + flattened = self._render_list(block.list).replace("\n", " ").strip() + if flattened: + parts.append(flattened) + elif kind == "table": + for row in block.table.grid: + values = [ + self._render_cell(slot.cell) if slot.kind == "origin" else "" + for slot in row + ] + if any(values): + parts.append(" / ".join(values)) + elif kind == "block_quote": + text = self._render_blocks(block.blocks or []).replace("\n", " ").strip() + if text: + parts.append(text) + elif kind == "code_block": + text = (block.text or "").strip() + if text: + fence = self._backtick_fence(text, 1) + parts.append(f"{fence}{text}{fence}") + elif kind != "rule": + raise RuntimeError(f"Unsupported AnyDoc table-cell block kind: {kind}") + return "
".join(line.strip() for line in "
".join(parts).splitlines() if line.strip()) + + @staticmethod + def _format_table_row(cells: Iterable[str]) -> str: + return "|" + "".join(f" {cell} |" for cell in cells) + + @staticmethod + def _table_is_single_cell(table: Any) -> bool: + return ( + len(table.grid) == 1 and len(table.grid[0]) == 1 and table.grid[0][0].kind == "origin" + ) + + def _number_notes(self) -> dict[str, int]: + notes = {note.id: note for note in self.document.notes if note.blocks} + order: list[str] = [] + seen: set[str] = set() + + def visit_inlines(inlines: Iterable[Any]) -> None: + for inline in inlines: + if inline.kind == "note_ref" and inline.note_id in notes: + if inline.note_id not in seen: + seen.add(inline.note_id) + order.append(inline.note_id) + visit_blocks(notes[inline.note_id].blocks) + elif inline.kind == "link": + visit_inlines(inline.content or []) + + def visit_blocks(blocks: Iterable[Any]) -> None: + for block in blocks: + if block.kind in {"heading", "paragraph"}: + visit_inlines(block.content or []) + elif block.kind == "list": + for item in block.list.items: + visit_blocks(item.blocks) + elif block.kind == "table": + for row in block.table.grid: + for slot in row: + if slot.kind == "origin": + visit_blocks(slot.cell.blocks) + elif block.kind == "block_quote": + visit_blocks(block.blocks or []) + + visit_blocks(self.document.blocks) + for note in self.document.notes: + if note.id in notes and note.id not in seen: + seen.add(note.id) + order.append(note.id) + return {note_id: index for index, note_id in enumerate(order, 1)} + + def _resolve_anchors(self) -> dict[str, _ResolvedAnchor]: + linked: set[str] = set() + headings: list[Any] = [] + anchors: list[str] = [] + + def visit_inlines(inlines: Iterable[Any]) -> None: + for inline in inlines: + if inline.kind == "link": + if inline.target.kind == "anchor": + linked.add(inline.target.value) + visit_inlines(inline.content or []) + elif inline.kind == "anchor" and inline.anchor: + anchors.append(inline.anchor) + + def visit_blocks(blocks: Iterable[Any]) -> None: + for block in blocks: + if block.kind == "heading": + headings.append(block) + visit_inlines(block.content or []) + elif block.kind == "paragraph": + visit_inlines(block.content or []) + elif block.kind == "list": + for item in block.list.items: + visit_blocks(item.blocks) + elif block.kind == "table": + for row in block.table.grid: + for slot in row: + if slot.kind == "origin": + visit_blocks(slot.cell.blocks) + elif block.kind == "block_quote": + visit_blocks(block.blocks or []) + + visit_blocks(self.document.blocks) + for note in self.document.notes: + visit_blocks(note.blocks) + + resolved: dict[str, _ResolvedAnchor] = {} + used: set[str] = set() + for heading in headings: + plain = self._plain_text(heading.content or []) + slug = self._claim_anchor(self._gfm_slug(plain), used) + heading_ids = [heading.anchor] if heading.anchor else [] + heading_ids.extend(self._inline_anchor_ids(heading.content or [])) + for anchor_id in heading_ids: + resolved.setdefault(anchor_id, _ResolvedAnchor(slug, False)) + for anchor_id in anchors: + if anchor_id in linked and anchor_id not in resolved: + fragment = self._claim_anchor(self._sanitize_anchor(anchor_id), used) + resolved[anchor_id] = _ResolvedAnchor(fragment, True) + return resolved + + def _plain_text(self, inlines: Iterable[Any]) -> str: + parts: list[str] = [] + for inline in inlines: + if inline.kind == "text": + parts.append(inline.text or "") + elif inline.kind == "link": + parts.append(self._plain_text(inline.content or [])) + elif inline.kind == "image": + parts.append(inline.alt or "") + elif inline.kind == "line_break": + parts.append(" ") + return "".join(parts) + + def _inline_anchor_ids(self, inlines: Iterable[Any]) -> list[str]: + ids: list[str] = [] + for inline in inlines: + if inline.kind == "anchor" and inline.anchor: + ids.append(inline.anchor) + elif inline.kind == "link": + ids.extend(self._inline_anchor_ids(inline.content or [])) + return ids + + @staticmethod + def _claim_anchor(base: str, used: set[str]) -> str: + if base not in used: + used.add(base) + return base + suffix = 1 + while f"{base}-{suffix}" in used: + suffix += 1 + claimed = f"{base}-{suffix}" + used.add(claimed) + return claimed + + @staticmethod + def _gfm_slug(text: str) -> str: + slug = "".join( + "-" if char == " " else char.lower() + for char in text.strip() + if char == " " or char == "-" or char == "_" or char.isalnum() + ) + return slug or "section" + + @staticmethod + def _sanitize_anchor(anchor: str) -> str: + sanitized = re.sub(r"[^a-z0-9_-]+", "-", anchor.lower()).strip("-") + return sanitized or "anchor" + + @staticmethod + def _marker_label(marker: str, number: int) -> str: + if marker in {"lower_alpha", "upper_alpha"}: + alpha = "" + remaining = max(number, 1) + while remaining: + remaining, offset = divmod(remaining - 1, 26) + alpha = chr(ord("a") + offset) + alpha + return f"{alpha.upper() if marker == 'upper_alpha' else alpha}." + if marker in {"lower_roman", "upper_roman"}: + remaining = max(number, 1) + result = "" + for unit, symbol in ( + (1000, "M"), + (900, "CM"), + (500, "D"), + (400, "CD"), + (100, "C"), + (90, "XC"), + (50, "L"), + (40, "XL"), + (10, "X"), + (9, "IX"), + (5, "V"), + (4, "IV"), + (1, "I"), + ): + while remaining >= unit: + result += symbol + remaining -= unit + return f"{result.lower() if marker == 'lower_roman' else result}." + raise RuntimeError(f"Unsupported AnyDoc list marker: {marker}") + + @staticmethod + def _format_url(url: str) -> str: + escaped = "".join( + "%7C" if char == "|" else "%3C" if char == "<" else "%3E" if char == ">" else char + for char in url + if ord(char) >= 32 and ord(char) != 127 + ) + return ( + f"<{escaped}>" if any(char.isspace() or char in "()" for char in escaped) else escaped + ) + + @staticmethod + def _backtick_fence(text: str, minimum: int) -> str: + longest = max((len(run) for run in re.findall(r"`+", text)), default=0) + return "`" * max(longest + 1, minimum) + + @staticmethod + def _escape_text( + text: str, + *, + context: str, + styled: bool = False, + trailing_active: bool = False, + in_label: bool = False, + at_line_start: bool = False, + ) -> str: + characters = list(text) + output: list[str] = [] + line_has_content = not at_line_start + for index, char in enumerate(characters): + if char == "\n": + output.append(char) + if context == "block": + line_has_content = False + continue + start_of_line = not line_has_content + if not char.isspace(): + line_has_content = True + next_char = characters[index + 1] if index + 1 < len(characters) else None + next_nonspace = trailing_active if next_char is None else not next_char.isspace() + later = characters[index + 1 :] + escape = False + if char == "\\": + escape = True + elif char == "]" and in_label: + escape = True + elif char == "`": + escape = styled or "`" in later + elif char == "*": + escape = styled or start_of_line or (next_nonspace and "*" in later) + elif char == "_": + previous_alnum = index > 0 and characters[index - 1].isalnum() + next_alnum = bool(next_char and next_char.isalnum()) + escape = styled or ( + next_nonspace and not (previous_alnum and next_alnum) and "_" in later + ) + elif char == "~": + escape = styled or (next_nonspace and "~" in later) + elif char == "[": + escape = in_label or "]" in later + elif char == "<": + escape = bool(next_char and (next_char.isalpha() or next_char in "/!?")) + elif char == "!": + escape = next_char is None and trailing_active + elif char == "|" and context == "table_cell": + escape = True + elif char in "#>" and start_of_line: + escape = True + elif char in "-+" and start_of_line: + escape = next_char is None or next_char.isspace() + if escape: + output.append("\\") + output.append(char) + return "".join(output) + + +class AnyDocParser(BaseParser): + """Parse the existing Office/EPUB formats through AnyDoc's document model.""" + + _SUPPORTED_EXTENSIONS = [".doc", ".docx", ".pptx", ".xls", ".xlsx", ".xlsm", ".epub"] + + def __init__(self, config: Optional[ParserConfig] = None): + from openviking.parse.parsers.markdown import MarkdownParser + + self.config = config or ParserConfig() + self._markdown_parser = MarkdownParser(config=self.config) + + @property + def supported_extensions(self) -> List[str]: + return list(self._SUPPORTED_EXTENSIONS) + + async def parse(self, source: Union[str, Path], instruction: str = "", **kwargs) -> ParseResult: + started = time.time() + path = Path(source) + if not path.is_file(): + raise FileNotFoundError(f"Document file not found: {path}") + + from openviking_cli.utils.storage import get_storage + + storage = get_storage() + resource_name = kwargs.get("resource_name") or kwargs.get("source_name") or path.stem + rendered = await asyncio.to_thread( + self._convert, + path, + resource_name=resource_name, + storage=storage, + ) + + markdown_kwargs = dict(kwargs) + caller_media_dirs = list(markdown_kwargs.pop("allowed_media_dirs", None) or []) + caller_media_dirs.append(storage.media_dir) + result = await self._markdown_parser.parse_content( + rendered.markdown, + source_path=str(path), + instruction=instruction, + base_dir=path.parent, + allowed_media_dirs=caller_media_dirs, + **markdown_kwargs, + ) + result.source_format = path.suffix.lower().lstrip(".") + result.parser_name = "AnyDocParser" + result.parser_version = "1.0" + result.parse_time = time.time() - started + result.warnings.extend(rendered.warnings) + result.meta.update( + { + "library": "firecrawl-anydoc", + "library_version": version("firecrawl-anydoc"), + "detected_format": rendered.detected_format, + "assets_referenced": rendered.assets_referenced, + "images_extracted": rendered.images_extracted, + "intermediate_markdown_length": len(rendered.markdown), + } + ) + return result + + def _convert(self, path: Path, *, resource_name: str, storage: Any) -> _RenderedDocument: + import anydoc + + data = path.read_bytes() + detected_format = anydoc.format_from_bytes(data) + selected_format = detected_format or anydoc.format_from_extension(path.suffix) + if selected_format is None: + raise anydoc.UnsupportedError(f"Unrecognized document format: {path.name}") + document = anydoc.to_document(data, selected_format) + renderer = _AnyDocMarkdownRenderer( + document, + source_format=path.suffix.lower().lstrip("."), + resource_name=resource_name, + storage=storage, + ) + markdown = renderer.render() + if not markdown.strip(): + raise anydoc.MalformedError(f"No meaningful content extracted from {path.name}") + return _RenderedDocument( + markdown=markdown, + detected_format=detected_format or selected_format, + warnings=renderer.warnings, + assets_referenced=len(renderer.assets_referenced), + images_extracted=renderer.images_extracted, + ) + + async def parse_content( + self, + content: str, + source_path: Optional[str] = None, + instruction: str = "", + **kwargs, + ) -> ParseResult: + raise NotImplementedError( + "AnyDocParser requires a document file path; string content is not supported" + ) diff --git a/openviking/parse/parsers/epub.py b/openviking/parse/parsers/epub.py deleted file mode 100644 index 6ecc76bbc0..0000000000 --- a/openviking/parse/parsers/epub.py +++ /dev/null @@ -1,215 +0,0 @@ -# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. -# SPDX-License-Identifier: AGPL-3.0 -""" -EPub (.epub) parser for OpenViking. - -Converts EPub e-books to Markdown then parses using MarkdownParser. -Inspired by microsoft/markitdown approach. -""" - -import asyncio -import html -import re -import zipfile -from html.parser import HTMLParser -from pathlib import Path -from typing import List, Optional, Union - -from openviking.parse.base import ParseResult -from openviking.parse.parsers.base_parser import BaseParser -from openviking_cli.utils.config.parser_config import ParserConfig -from openviking_cli.utils.logger import get_logger - -logger = get_logger(__name__) - - -class _EPubMarkdownParser(HTMLParser): - """Convert HTML fragments to simple markdown without regex tag stripping.""" - - _HEADER_PREFIX = {"h1": "# ", "h2": "## ", "h3": "### ", "h4": "#### "} - - def __init__(self): - super().__init__(convert_charrefs=True) - self._parts: list[str] = [] - self._ignored_tag_stack = 0 - - def handle_starttag(self, tag: str, attrs): - normalized_tag = tag.lower() - if normalized_tag in {"script", "style"}: - self._ignored_tag_stack += 1 - return - if self._ignored_tag_stack: - return - if normalized_tag in self._HEADER_PREFIX: - self._ensure_block_break() - self._parts.append(self._HEADER_PREFIX[normalized_tag]) - elif normalized_tag in {"strong", "b"}: - self._parts.append("**") - elif normalized_tag in {"em", "i"}: - self._parts.append("*") - elif normalized_tag == "li": - self._parts.append("\n- ") - elif normalized_tag == "br": - self._parts.append("\n") - elif normalized_tag in {"p", "div", "section", "article"}: - self._ensure_block_break() - - def handle_endtag(self, tag: str): - normalized_tag = tag.lower() - if normalized_tag in {"script", "style"}: - if self._ignored_tag_stack: - self._ignored_tag_stack -= 1 - return - if self._ignored_tag_stack: - return - if normalized_tag in self._HEADER_PREFIX or normalized_tag in { - "p", - "div", - "section", - "article", - }: - self._parts.append("\n\n") - elif normalized_tag in {"strong", "b"}: - self._parts.append("**") - elif normalized_tag in {"em", "i"}: - self._parts.append("*") - - def handle_data(self, data: str): - if not self._ignored_tag_stack: - self._parts.append(data) - - def get_markdown(self) -> str: - return "".join(self._parts) - - def _ensure_block_break(self): - if self._parts and not self._parts[-1].endswith("\n"): - self._parts.append("\n\n") - - -class EPubParser(BaseParser): - """ - EPub e-book parser for OpenViking. - - Supports: .epub - - Converts EPub e-books to Markdown using ebooklib (if available) - or falls back to manual extraction, then delegates to MarkdownParser. - """ - - def __init__(self, config: Optional[ParserConfig] = None): - """Initialize EPub parser.""" - from openviking.parse.parsers.markdown import MarkdownParser - - self._md_parser = MarkdownParser(config=config) - self.config = config or ParserConfig() - - @property - def supported_extensions(self) -> List[str]: - return [".epub"] - - async def parse(self, source: Union[str, Path], instruction: str = "", **kwargs) -> ParseResult: - """Parse EPub e-book from file path.""" - path = Path(source) - - if path.exists(): - markdown_content = await asyncio.to_thread(self._convert_to_markdown, path) - result = await self._md_parser.parse_content( - markdown_content, source_path=str(path), instruction=instruction, **kwargs - ) - else: - result = await self._md_parser.parse_content( - str(source), instruction=instruction, **kwargs - ) - result.source_format = "epub" - result.parser_name = "EPubParser" - return result - - async def parse_content( - self, content: str, source_path: Optional[str] = None, instruction: str = "", **kwargs - ) -> ParseResult: - """Parse content - delegates to MarkdownParser.""" - result = await self._md_parser.parse_content(content, source_path, **kwargs) - result.source_format = "epub" - result.parser_name = "EPubParser" - return result - - def _convert_to_markdown(self, path: Path) -> str: - """Convert EPub e-book to Markdown string.""" - # Try using ebooklib first - try: - import ebooklib - from ebooklib import epub - - return self._convert_with_ebooklib(path, ebooklib, epub) - except ImportError: - pass - - # Fall back to manual extraction - return self._convert_manual(path) - - def _convert_with_ebooklib(self, path: Path, ebooklib, epub) -> str: - """Convert EPub using ebooklib.""" - book = epub.read_epub(path) - markdown_parts = [] - - title = self._get_metadata(book, "title") - author = self._get_metadata(book, "creator") - - if title: - markdown_parts.append(f"# {title}") - if author: - markdown_parts.append(f"**Author:** {author}") - - for item in book.get_items(): - if item.get_type() == ebooklib.ITEM_DOCUMENT: - content = item.get_content().decode("utf-8", errors="ignore") - md_content = self._html_to_markdown(content) - if md_content.strip(): - markdown_parts.append(md_content) - - return "\n\n".join(markdown_parts) - - def _get_metadata(self, book, key: str) -> str: - """Get metadata from EPub book.""" - try: - metadata = book.get_metadata("DC", key) - if metadata: - return metadata[0][0] - except Exception: - pass - return "" - - def _convert_manual(self, path: Path) -> str: - """Convert EPub manually using zipfile and HTML parsing.""" - markdown_parts = [] - - with zipfile.ZipFile(path, "r") as zf: - html_files = [f for f in zf.namelist() if f.endswith((".html", ".xhtml", ".htm"))] - - for html_file in sorted(html_files): - try: - content = zf.read(html_file).decode("utf-8", errors="ignore") - md_content = self._html_to_markdown(content) - if md_content.strip(): - markdown_parts.append(md_content) - except Exception as e: - logger.warning(f"Failed to process {html_file}: {e}") - - return ( - "\n\n".join(markdown_parts) - if markdown_parts - else "# EPub Content\n\nUnable to extract content." - ) - - def _html_to_markdown(self, html_content: str) -> str: - """Simple HTML to markdown conversion.""" - parser = _EPubMarkdownParser() - parser.feed(html_content) - parser.close() - markdown = html.unescape(parser.get_markdown()) - - # Normalize whitespace - markdown = re.sub(r"\n\s*\n", "\n\n", markdown) - markdown = re.sub(r"[ \t]+", " ", markdown) - - return markdown.strip() diff --git a/openviking/parse/parsers/excel.py b/openviking/parse/parsers/excel.py deleted file mode 100644 index d8c610c77a..0000000000 --- a/openviking/parse/parsers/excel.py +++ /dev/null @@ -1,440 +0,0 @@ -# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. -# SPDX-License-Identifier: AGPL-3.0 -""" -Excel (.xlsx/.xls/.xlsm) parser for OpenViking. - -Converts Excel spreadsheets to Markdown then parses using MarkdownParser. -Inspired by microsoft/markitdown approach. -""" - -import asyncio -import concurrent.futures -import time -from dataclasses import asdict, fields -from functools import partial -from multiprocessing import get_context -from pathlib import Path -from typing import Any, Dict, List, Optional, Union - -from openviking.parse.base import NodeType, ParseResult, ResourceNode, create_parse_result -from openviking.parse.parsers.base_parser import BaseParser -from openviking_cli.utils.config.parser_config import ExcelConfig, ParserConfig -from openviking_cli.utils.logger import get_logger - -logger = get_logger(__name__) - -_EXCEL_LAYOUT_EXECUTOR: Optional[concurrent.futures.ProcessPoolExecutor] = None -_EXCEL_LAYOUT_EXECUTOR_WORKERS: Optional[int] = None - -# Non-configurable process-pool internals. -_EXCEL_PROCESS_POOL_START_METHOD = "spawn" -_EXCEL_PROCESS_POOL_MIN_BYTES = 200_000 -_EXCEL_PROCESS_POOL_TIMEOUT_S = 120.0 - - -def _get_excel_layout_executor(workers: int) -> concurrent.futures.ProcessPoolExecutor: - global _EXCEL_LAYOUT_EXECUTOR, _EXCEL_LAYOUT_EXECUTOR_WORKERS - workers = max(1, int(workers)) - if _EXCEL_LAYOUT_EXECUTOR is None: - _EXCEL_LAYOUT_EXECUTOR = concurrent.futures.ProcessPoolExecutor( - max_workers=workers, - mp_context=get_context(_EXCEL_PROCESS_POOL_START_METHOD), - ) - _EXCEL_LAYOUT_EXECUTOR_WORKERS = workers - logger.info( - f"[ExcelParserProcessPool] Started process pool " - f"workers={workers} start_method={_EXCEL_PROCESS_POOL_START_METHOD}" - ) - elif _EXCEL_LAYOUT_EXECUTOR_WORKERS != workers: - logger.warning( - "[ExcelParserProcessPool] Ignoring process_pool_workers=%s; " - "pool already started with workers=%s", - workers, - _EXCEL_LAYOUT_EXECUTOR_WORKERS, - ) - return _EXCEL_LAYOUT_EXECUTOR - - -def _build_excel_layout_in_process( - *, - path_str: str, - temp_uri: str, - instruction: str, - layout_kwargs: Dict[str, Any], - config_dict: Dict[str, Any], - max_rows_per_sheet: int, -) -> Dict[str, Any]: - """CPU-only child process worker. - - The worker must not touch VikingFS, DB, queues, or RequestContext. It only - converts Excel to markdown and computes MarkdownParser layout ops. - """ - import asyncio - - from openviking.parse.parsers.excel import ExcelParser - from openviking_cli.utils.config.parser_config import ParserConfig - - started = time.perf_counter() - path = Path(path_str) - allowed_config_fields = {field.name for field in fields(ParserConfig)} - filtered_config = { - key: value for key, value in config_dict.items() if key in allowed_config_fields - } - parser = ExcelParser( - config=ParserConfig.from_dict(filtered_config), - max_rows_per_sheet=max_rows_per_sheet, - ) - - convert_started = time.perf_counter() - if path.suffix.lower() == ".xls": - markdown_content = parser._convert_xls_to_markdown(path) - else: - import openpyxl - - markdown_content = parser._convert_to_markdown(path, openpyxl) - convert_s = time.perf_counter() - convert_started - - layout_started = time.perf_counter() - layout = asyncio.run( - parser._md_parser._compute_layout( - markdown_content, - temp_uri, - source_path=str(path), - instruction=instruction, - **layout_kwargs, - ) - ) - layout_s = time.perf_counter() - layout_started - - return { - "layout": layout, - "convert_s": convert_s, - "layout_s": layout_s, - "total_s": time.perf_counter() - started, - "markdown_chars": len(markdown_content), - "layout_ops": len(layout.ops), - "layout_write_ops": sum(1 for op in layout.ops if op.kind == "write"), - "layout_mkdir_ops": sum(1 for op in layout.ops if op.kind == "mkdir"), - } - - -class ExcelParser(BaseParser): - """ - Excel spreadsheet parser for OpenViking. - - Supports: .xlsx, .xls, .xlsm - - Converts Excel spreadsheets to Markdown using openpyxl, - then delegates to MarkdownParser for tree structure creation. - """ - - def __init__( - self, config: Optional[ParserConfig] = None, max_rows_per_sheet: int = 1000 - ): - """ - Initialize Excel parser. - - Args: - config: Parser configuration (prefer ExcelConfig for process-pool knobs) - max_rows_per_sheet: Maximum rows to process per sheet (0 = unlimited) - """ - from openviking.parse.parsers.markdown import MarkdownParser - - self._md_parser = MarkdownParser(config=config) - self.config = config or ExcelConfig() - self.max_rows_per_sheet = max_rows_per_sheet - - def _process_pool_enabled(self) -> bool: - return bool(getattr(self.config, "enable_process_pool", False)) - - def _process_pool_workers(self) -> int: - return max(1, int(getattr(self.config, "process_pool_workers", 2) or 2)) - - @property - def supported_extensions(self) -> List[str]: - return [".xlsx", ".xls", ".xlsm"] - - async def parse(self, source: Union[str, Path], instruction: str = "", **kwargs) -> ParseResult: - """Parse Excel spreadsheet from file path.""" - path = Path(source) - - if path.exists(): - result = await self._parse_existing_path(path, instruction=instruction, **kwargs) - else: - result = await self._md_parser.parse_content( - str(source), instruction=instruction, **kwargs - ) - result.source_format = path.suffix.lstrip(".") if path.exists() else "xlsx" - result.parser_name = "ExcelParser" - return result - - async def _parse_existing_path( - self, path: Path, instruction: str = "", **kwargs - ) -> ParseResult: - if self._should_use_process_pool(path, kwargs): - try: - return await self._parse_existing_path_process_pool( - path, instruction=instruction, **kwargs - ) - except Exception as exc: - logger.warning( - f"[ExcelParserProcessPool] Falling back to in-process parse: {exc}", - exc_info=True, - ) - - # Use xlrd for legacy .xls, openpyxl for .xlsx/.xlsm - if path.suffix.lower() == ".xls": - markdown_content = await asyncio.to_thread(self._convert_xls_to_markdown, path) - else: - import openpyxl - - markdown_content = await asyncio.to_thread(self._convert_to_markdown, path, openpyxl) - return await self._md_parser.parse_content( - markdown_content, source_path=str(path), instruction=instruction, **kwargs - ) - - def _should_use_process_pool(self, path: Path, kwargs: Dict[str, Any]) -> bool: - if not self._process_pool_enabled(): - return False - if path.suffix.lower() == ".xls": - return False - if kwargs.get("enable_link_rewrite") or kwargs.get("base_dir") or kwargs.get("allowed_media_dirs"): - logger.debug("[ExcelParserProcessPool] Disabled for link/media rewrite parse") - return False - try: - return path.stat().st_size >= _EXCEL_PROCESS_POOL_MIN_BYTES - except OSError: - return False - - async def _parse_existing_path_process_pool( - self, path: Path, instruction: str = "", **kwargs - ) -> ParseResult: - parse_started = time.perf_counter() - loop = asyncio.get_running_loop() - temp_uri = self._md_parser._create_temp_uri() - layout_kwargs: Dict[str, Any] = { - key: value - for key, value in kwargs.items() - if key in {"resource_name", "source_name"} and isinstance(value, str) - } - if isinstance(kwargs.get("split_content"), bool): - layout_kwargs["split_content"] = kwargs["split_content"] - future = loop.run_in_executor( - _get_excel_layout_executor(self._process_pool_workers()), - partial( - _build_excel_layout_in_process, - path_str=str(path), - temp_uri=temp_uri, - instruction=instruction, - layout_kwargs=layout_kwargs, - config_dict=asdict(self.config), - max_rows_per_sheet=self.max_rows_per_sheet, - ), - ) - - worker_started = time.perf_counter() - worker_result = await asyncio.wait_for( - future, timeout=_EXCEL_PROCESS_POOL_TIMEOUT_S - ) - worker_s = time.perf_counter() - worker_started - layout = worker_result["layout"] - - self._md_parser._rewrite_ctx = { - "enabled": False, - "source_path": str(path), - "doc_name": layout.doc_name, - "root_dir": layout.root_dir, - "import_root": None, - "base_dir": None, - "allowed_media_dirs": None, - } - try: - apply_started = time.perf_counter() - await self._md_parser._apply_layout(layout) - apply_s = time.perf_counter() - apply_started - finally: - self._md_parser._rewrite_ctx = None - - parse_time = time.perf_counter() - parse_started - logger.info( - f"[ExcelParserProcessPool] path={path} total={parse_time:.3f}s " - f"worker_wall={worker_s:.3f}s convert={worker_result.get('convert_s', -1.0):.3f}s " - f"layout={worker_result.get('layout_s', -1.0):.3f}s apply={apply_s:.3f}s " - f"chars={worker_result.get('markdown_chars')} ops={worker_result.get('layout_ops')} " - f"writes={worker_result.get('layout_write_ops')} mkdirs={worker_result.get('layout_mkdir_ops')}" - ) - - root = ResourceNode( - type=NodeType.ROOT, - title=layout.doc_title, - level=0, - meta=layout.meta.get("frontmatter", {}), - ) - result = create_parse_result( - root=root, - source_path=str(path), - source_format="markdown", - parser_name="MarkdownParser", - parse_time=parse_time, - meta=layout.meta, - warnings=layout.warnings, - ) - result.temp_dir_path = layout.temp_uri - return result - - async def parse_content( - self, content: str, source_path: Optional[str] = None, instruction: str = "", **kwargs - ) -> ParseResult: - """Parse content - delegates to MarkdownParser.""" - result = await self._md_parser.parse_content(content, source_path, **kwargs) - result.source_format = "xlsx" - result.parser_name = "ExcelParser" - return result - - def _convert_xls_to_markdown(self, path: Path) -> str: - """Convert legacy .xls spreadsheet to Markdown using xlrd.""" - import xlrd - - # formatting_info=True enables xlrd to detect date cells via XL_CELL_DATE - # instead of reporting them as XL_CELL_NUMBER with raw float serials - wb = xlrd.open_workbook(str(path), formatting_info=True, on_demand=True) - try: - return self._build_xls_markdown(wb, path, xlrd) - finally: - wb.release_resources() - - def _build_xls_markdown(self, wb, path: Path, xlrd) -> str: - """Build markdown from xlrd workbook.""" - markdown_parts = [] - markdown_parts.append(f"# {path.stem}") - markdown_parts.append(f"**Sheets:** {wb.nsheets}") - - for sheet_idx in range(wb.nsheets): - sheet = wb.sheet_by_index(sheet_idx) - parts = [f"## Sheet: {sheet.name}"] - - if sheet.nrows == 0 or sheet.ncols == 0: - parts.append("*Empty sheet*") - markdown_parts.append("\n\n".join(parts)) - continue - - parts.append(f"**Dimensions:** {sheet.nrows} rows × {sheet.ncols} columns") - - rows_to_process = sheet.nrows - if self.max_rows_per_sheet > 0: - rows_to_process = min(sheet.nrows, self.max_rows_per_sheet) - - rows = [] - for row_idx in range(rows_to_process): - row_data = [] - for col_idx in range(sheet.ncols): - row_data.append(self._format_xls_cell(sheet.cell(row_idx, col_idx), wb, xlrd)) - rows.append(row_data) - - if rows: - from openviking.parse.base import format_table_to_markdown - - parts.append(format_table_to_markdown(rows, has_header=True)) - - if self.max_rows_per_sheet > 0 and sheet.nrows > self.max_rows_per_sheet: - parts.append( - f"\n*... {sheet.nrows - self.max_rows_per_sheet} more rows truncated ...*" - ) - - markdown_parts.append("\n\n".join(parts)) - - return "\n\n".join(markdown_parts) - - @staticmethod - def _format_xls_cell(cell, wb, xlrd) -> str: - """Format a single xlrd cell value with proper type handling.""" - if cell.ctype == xlrd.XL_CELL_EMPTY or cell.ctype == xlrd.XL_CELL_BLANK: - return "" - if cell.ctype == xlrd.XL_CELL_DATE: - try: - dt = xlrd.xldate_as_tuple(cell.value, wb.datemode) - # Include time component if non-zero - if dt[3] or dt[4] or dt[5]: - return ( - f"{dt[0]:04d}-{dt[1]:02d}-{dt[2]:02d} {dt[3]:02d}:{dt[4]:02d}:{dt[5]:02d}" - ) - return f"{dt[0]:04d}-{dt[1]:02d}-{dt[2]:02d}" - except Exception: - return str(cell.value) - if cell.ctype == xlrd.XL_CELL_BOOLEAN: - return "TRUE" if cell.value else "FALSE" - if cell.ctype == xlrd.XL_CELL_ERROR: - # xlrd error code map - error_map = { - 0x00: "#NULL!", - 0x07: "#DIV/0!", - 0x0F: "#VALUE!", - 0x17: "#REF!", - 0x1D: "#NAME?", - 0x24: "#NUM!", - 0x2A: "#N/A", - } - return error_map.get(cell.value, f"#ERR({cell.value})") - if cell.ctype == xlrd.XL_CELL_NUMBER: - # Display integers without trailing .0 - if cell.value == int(cell.value): - return str(int(cell.value)) - return str(cell.value) - # XL_CELL_TEXT or fallback - return str(cell.value) if cell.value is not None else "" - - def _convert_to_markdown(self, path: Path, openpyxl) -> str: - """Convert Excel spreadsheet to Markdown string.""" - wb = openpyxl.load_workbook(path, data_only=True) - - markdown_parts = [] - markdown_parts.append(f"# {path.stem}") - markdown_parts.append(f"**Sheets:** {len(wb.sheetnames)}") - - for sheet_name in wb.sheetnames: - sheet = wb[sheet_name] - sheet_content = self._convert_sheet(sheet, sheet_name) - markdown_parts.append(sheet_content) - - return "\n\n".join(markdown_parts) - - def _convert_sheet(self, sheet, sheet_name: str) -> str: - """Convert a single sheet to markdown.""" - parts = [] - parts.append(f"## Sheet: {sheet_name}") - - max_row = sheet.max_row - max_col = sheet.max_column - - if max_row == 0 or max_col == 0: - parts.append("*Empty sheet*") - return "\n\n".join(parts) - - parts.append(f"**Dimensions:** {max_row} rows × {max_col} columns") - - rows_to_process = max_row - if self.max_rows_per_sheet > 0: - rows_to_process = min(max_row, self.max_rows_per_sheet) - - rows = [] - for _row_idx, row in enumerate( - sheet.iter_rows(min_row=1, max_row=rows_to_process, values_only=True), 1 - ): - row_data = [] - for cell in row: - if cell is None: - row_data.append("") - else: - row_data.append(str(cell)) - rows.append(row_data) - - if rows: - from openviking.parse.base import format_table_to_markdown - - table_md = format_table_to_markdown(rows, has_header=True) - parts.append(table_md) - - if self.max_rows_per_sheet > 0 and max_row > self.max_rows_per_sheet: - parts.append(f"\n*... {max_row - self.max_rows_per_sheet} more rows truncated ...*") - - return "\n\n".join(parts) diff --git a/openviking/parse/parsers/legacy_doc.py b/openviking/parse/parsers/legacy_doc.py deleted file mode 100644 index 11b48a5194..0000000000 --- a/openviking/parse/parsers/legacy_doc.py +++ /dev/null @@ -1,408 +0,0 @@ -# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. -# SPDX-License-Identifier: AGPL-3.0 -""" -Legacy Word document (.doc) parser for OpenViking. - -Extracts text from OLE2 compound binary .doc files using olefile, -then delegates to MarkdownParser for tree structure creation. -""" - -import asyncio -import struct -import zipfile -from pathlib import Path -from typing import List, Optional, Union - -from openviking.parse.base import ParseResult -from openviking.parse.parsers.base_parser import BaseParser -from openviking_cli.utils.config.parser_config import ParserConfig -from openviking_cli.utils.logger import get_logger - -logger = get_logger(__name__) - - -# Max stream size to read (50MB) — prevents DoS from crafted files -_MAX_STREAM_SIZE = 50 * 1024 * 1024 -# Max character count sanity cap for ccpText -_MAX_CCP_TEXT = 10_000_000 -_ZIP_SIGNATURES = (b"PK\x03\x04", b"PK\x05\x06", b"PK\x07\x08") - - -class LegacyDocParser(BaseParser): - """ - Legacy .doc (OLE2 binary) parser. - - Extracts text content from Word 97-2003 (.doc) files using olefile - to read the WordDocument and table streams, then delegates to - MarkdownParser for tree structure. - """ - - def __init__(self, config: Optional[ParserConfig] = None): - from openviking.parse.parsers.markdown import MarkdownParser - - self._md_parser = MarkdownParser(config=config) - self.config = config or ParserConfig() - - @property - def supported_extensions(self) -> List[str]: - return [".doc"] - - async def parse(self, source: Union[str, Path], instruction: str = "", **kwargs) -> ParseResult: - """Parse legacy .doc file.""" - path = Path(source) - - if path.exists(): - if self._has_zip_signature(path): - package_type = await asyncio.to_thread(self._classify_zip_package, path) - if package_type == "docx": - # python-docx writes OOXML regardless of the filename suffix. This - # also occurs in real uploads whose extension says .doc while the - # payload is a modern Word ZIP package. Never pass those bytes to - # the legacy UTF-16 fallback: it turns ZIP data into plausible but - # meaningless Unicode and silently persists corrupted Markdown. - from openviking.parse.parsers.word import WordParser - - logger.info( - "Detected OOXML Word content in %s; routing to WordParser", - path.name, - ) - return await WordParser(config=self.config).parse( - path, - instruction=instruction, - **kwargs, - ) - - raise ValueError(f"{path.name} is a ZIP package, not a legacy Word .doc file") - - text = await asyncio.to_thread(self._extract_text, path) - result = await self._md_parser.parse_content( - text, source_path=str(path), instruction=instruction, **kwargs - ) - else: - result = await self._md_parser.parse_content( - str(source), instruction=instruction, **kwargs - ) - result.source_format = "doc" - result.parser_name = "LegacyDocParser" - return result - - @staticmethod - def _has_zip_signature(path: Path) -> bool: - """Return whether the payload starts with a recognized ZIP signature.""" - with path.open("rb") as file_obj: - return file_obj.read(4) in _ZIP_SIGNATURES - - @staticmethod - def _classify_zip_package(path: Path) -> str: - """Identify a ZIP-backed Word package without extracting archive members.""" - try: - with zipfile.ZipFile(path) as archive: - try: - archive.getinfo("[Content_Types].xml") - archive.getinfo("word/document.xml") - except KeyError: - return "zip" - return "docx" - except (zipfile.BadZipFile, zipfile.LargeZipFile): - # A payload with ZIP magic is still not a legacy OLE .doc. Classify - # malformed archives as generic ZIP so callers fail closed instead of - # feeding binary bytes into the permissive legacy text fallback. - return "zip" - - async def parse_content( - self, content: str, source_path: Optional[str] = None, instruction: str = "", **kwargs - ) -> ParseResult: - """Parse content string — delegates to MarkdownParser.""" - result = await self._md_parser.parse_content( - content, source_path, instruction=instruction, **kwargs - ) - result.source_format = "doc" - result.parser_name = "LegacyDocParser" - return result - - def _extract_text(self, path: Path) -> str: - """ - Extract text from a legacy .doc OLE2 file. - - Reads the WordDocument stream and uses the FIB (File Information Block) - to locate text in the document body. Falls back to raw byte scanning - if structured extraction fails. - """ - import olefile - - try: - ole = olefile.OleFileIO(str(path)) - except Exception as e: - logger.warning(f"Failed to open .doc as OLE file: {e}") - return self._fallback_extract(path) - - try: - return self._extract_from_ole(ole) - except Exception as e: - logger.warning(f"Structured OLE extraction failed, using fallback: {e}") - return self._fallback_extract(path) - finally: - ole.close() - - @staticmethod - def _read_ole_stream(ole, stream_name: str) -> bytes: - """Read an OLE stream with size cap to prevent DoS.""" - stream = ole.openstream(stream_name) - data = stream.read(_MAX_STREAM_SIZE + 1) - if len(data) > _MAX_STREAM_SIZE: - raise ValueError(f"OLE stream '{stream_name}' exceeds {_MAX_STREAM_SIZE} bytes") - return data - - def _extract_from_ole(self, ole) -> str: - """ - Extract text from OLE streams using the Word Binary File Format. - - Reads the FIB to determine if text is stored as UTF-16 or compressed - (CP1252), then extracts the document body text from the appropriate - stream (WordDocument or table stream). - """ - if not ole.exists("WordDocument"): - raise ValueError("No WordDocument stream found") - - word_doc = self._read_ole_stream(ole, "WordDocument") - - # Minimum FIB size: need at least 0x01A8 bytes for Word 97+ FIB fields - if len(word_doc) < 0x01A8: - raise ValueError(f"WordDocument stream too small ({len(word_doc)} bytes)") - - # Check FIB version (nFib at offset 0x0002) — require Word 97+ (0x00C1+) - nfib = struct.unpack_from(" len(table_data): - return self._simple_text_extract(word_doc, ccp_text) - - return self._extract_via_clx(word_doc, table_data, fc_clx, lcb_clx, ccp_text) - - def _simple_text_extract(self, word_doc: bytes, ccp_text: int) -> str: - """ - Simple text extraction using FIB text offset. - - The main document text starts at offset 0x0800 in the WordDocument stream - for most Word 97+ files. Tries UTF-16LE first; falls back to CP1252 if - the stream is too small for UTF-16. - """ - text_start = 0x0800 # Standard text start offset - - if text_start >= len(word_doc): - raise ValueError("WordDocument stream too small for text extraction") - - # Try UTF-16LE first (2 bytes per char) - if ccp_text * 2 + text_start <= len(word_doc): - end = text_start + ccp_text * 2 - raw = word_doc[text_start:end] - text = raw.decode("utf-16-le", errors="replace") - # Sanity: if mostly printable, it's likely correct - if ( - sum(1 for c in text[:200] if c.isprintable() or c in "\n\r\t") - > len(text[:200]) * 0.5 - ): - return self._clean_word_text(text) - - # Fall back to CP1252 single-byte - end = min(text_start + ccp_text, len(word_doc)) - raw = word_doc[text_start:end] - return self._clean_word_text(self._decode_cp1252(raw)) - - def _extract_via_clx( - self, - word_doc: bytes, - table_data: bytes, - fc_clx: int, - lcb_clx: int, - ccp_text: int, - ) -> str: - """ - Extract text using the Clx (piece table) structure. - - The Clx contains a PiecePLC that maps character positions to file offsets, - allowing reconstruction of the document text even when pieces are scattered. - """ - clx = table_data[fc_clx : fc_clx + lcb_clx] - pos = 0 - text_parts = [] - chars_extracted = 0 - - # Skip any Grpprl (type 0x01) entries in the Clx - while pos < len(clx) and clx[pos] == 0x01: - if pos + 3 > len(clx): - break - cb = struct.unpack_from("= len(clx) or clx[pos] != 0x02: - return self._simple_text_extract(word_doc, ccp_text) - - pos += 1 # skip type byte - if pos + 4 > len(clx): - return self._simple_text_extract(word_doc, ccp_text) - - lcb_pcd = struct.unpack_from(" len(clx): - return self._simple_text_extract(word_doc, ccp_text) - - # Calculate number of pieces: (lcb_pcd - 4) / (4 + 8) per piece, - # but CPs are (n+1)*4 bytes + n*8 bytes = lcb_pcd - # So: 4*(n+1) + 8*n = lcb_pcd → 12n + 4 = lcb_pcd → n = (lcb_pcd - 4) / 12 - n_pieces = (lcb_pcd - 4) // 12 - if n_pieces <= 0: - return self._simple_text_extract(word_doc, ccp_text) - - # Read character positions (n+1 values) - cps = [] - for i in range(n_pieces + 1): - offset = pcd_start + i * 4 - if offset + 4 > len(clx): - break - cps.append(struct.unpack_from("= ccp_text: - break - - pcd_offset = pcd_array_start + i * 8 - if pcd_offset + 8 > len(clx): - break - - # PCD: 2 bytes flags, 4 bytes fc, 2 bytes prm - fc_value = struct.unpack_from(" {len(word_doc)})" - ) - else: - # UTF-16LE - byte_offset = fc_real - byte_end = byte_offset + piece_char_count * 2 - if byte_end <= len(word_doc): - raw = word_doc[byte_offset:byte_end] - text_parts.append(raw.decode("utf-16-le", errors="replace")) - else: - logger.warning( - f"Piece {i} extends beyond stream ({byte_end} > {len(word_doc)})" - ) - - chars_extracted += piece_char_count - - result = self._clean_word_text("".join(text_parts)) - if not result.strip(): - return self._simple_text_extract(word_doc, ccp_text) - return result - - @staticmethod - def _decode_cp1252(data: bytes) -> str: - """Decode CP1252 bytes to string.""" - return data.decode("cp1252", errors="replace") - - @staticmethod - def _clean_word_text(text: str) -> str: - """Normalize Word control characters to readable equivalents.""" - text = text.replace("\r\n", "\n").replace("\r", "\n") - # \x07 = cell/row end, \x0B = soft line break, \x0C = section break - text = text.replace("\x07", "\t").replace("\x0b", "\n").replace("\x0c", "\n\n") - return text - - def _fallback_extract(self, path: Path) -> str: - """ - Last-resort text extraction by scanning raw bytes for readable text runs. - - Tries UTF-16LE decoding first (common in .doc), then falls back to CP1252. - """ - # Cap read size to prevent DoS from large files - with open(path, "rb") as f: - raw = f.read(_MAX_STREAM_SIZE) - - # Try to find UTF-16LE text (every other byte is often 0x00 for ASCII) - try: - decoded = raw.decode("utf-16-le", errors="ignore") - # Filter to printable text runs - lines = [] - current = [] - for ch in decoded: - if ch.isprintable() or ch in "\n\t": - current.append(ch) - else: - if len(current) > 3: - lines.append("".join(current)) - current = [] - if current and len(current) > 3: - lines.append("".join(current)) - text = "\n".join(lines) - if len(text) > 50: - return text - except Exception: - pass - - # Fall back to CP1252 - text = raw.decode("cp1252", errors="replace") - lines = [] - current = [] - for ch in text: - if ch.isprintable() or ch in "\n\t": - current.append(ch) - else: - if len(current) > 3: - lines.append("".join(current)) - current = [] - if current and len(current) > 3: - lines.append("".join(current)) - return "\n".join(lines) diff --git a/openviking/parse/parsers/markdown.py b/openviking/parse/parsers/markdown.py index 4d267aa54a..7400cc15dd 100644 --- a/openviking/parse/parsers/markdown.py +++ b/openviking/parse/parsers/markdown.py @@ -19,7 +19,6 @@ import asyncio import hashlib -import io import os import re import time @@ -29,6 +28,7 @@ from openviking.parse.accessors.mime_types import IANA_MEDIA_TYPE_TO_EXTENSION from openviking.parse.base import NodeType, ParseResult, ResourceNode, create_parse_result +from openviking.parse.image_validation import is_valid_image from openviking.parse.parsers.base_parser import BaseParser from openviking.parse.parsers.code.ast.providers import supports_code_skeleton from openviking.parse.parsers.constants import ( @@ -160,14 +160,6 @@ class MarkdownParser(BaseParser): MAX_TOKENS_PER_CHAR = 0.7 MAX_MERGED_FILENAME_LENGTH = 32 # Maximum length for merged section filenames - # Image validation constants - IMAGE_MIN_SIDE = 14 # Minimum width/height in pixels (exclusive) - IMAGE_MIN_PIXELS = 196 # Minimum width * height - IMAGE_MAX_PIXELS = 36000000 # Maximum width * height - IMAGE_MIN_ASPECT_RATIO = 1 / 150 # Minimum width/height ratio - IMAGE_MAX_ASPECT_RATIO = 150 # Maximum width/height ratio - IMAGE_MAX_FILE_BYTES = 10 * 1024 * 1024 # Local file path limit: 10 MB - def __init__( self, extract_frontmatter: Optional[bool] = None, @@ -540,7 +532,7 @@ async def _has_ingestable_local_image( seen.add(resolved) try: image_bytes = await asyncio.to_thread(resolved.read_bytes) - if await asyncio.to_thread(self._is_valid_image, image_bytes, resolved): + if await asyncio.to_thread(is_valid_image, image_bytes, resolved): return True except Exception: continue @@ -779,7 +771,7 @@ async def _ingest_local_images( # Validate pixel size and file size; skip non-compliant images if not await asyncio.to_thread( - self._is_valid_image, image_bytes, resolved_path + is_valid_image, image_bytes, resolved_path ): continue @@ -891,63 +883,6 @@ def _resolve_image_path( logger.warning(f"[MarkdownParser] Cannot resolve image path: {path_str}") return None - def _is_valid_image(self, image_bytes: bytes, source_path: Path) -> bool: - """ - Validate an image's pixel dimensions and file size. - - Requirements: - - Width > 14px and height > 14px - - Width * height within [196, 36000000] - - Aspect ratio (width/height) within [1/150, 150] - - File size (local path) <= 10 MB - - Args: - image_bytes: Raw image bytes - source_path: Original image path (for logging) - - Returns: - True if the image satisfies all requirements, otherwise False - """ - # File size check (local file path limit: 10 MB) - if len(image_bytes) > self.IMAGE_MAX_FILE_BYTES: - logger.warning(f"[MarkdownParser] Image exceeds 10MB, skipping: {source_path}") - return False - - # Pixel size check - try: - from PIL import Image - - with Image.open(io.BytesIO(image_bytes)) as img: - width, height = img.size - except Exception as e: - logger.warning( - f"[MarkdownParser] Cannot read image dimensions, skipping {source_path}: {e}" - ) - return False - - if width <= self.IMAGE_MIN_SIDE or height <= self.IMAGE_MIN_SIDE: - logger.warning( - f"[MarkdownParser] Image side too small ({width}x{height}), skipping: {source_path}" - ) - return False - - pixels = width * height - if pixels < self.IMAGE_MIN_PIXELS or pixels > self.IMAGE_MAX_PIXELS: - logger.warning( - f"[MarkdownParser] Image pixel count out of range ({pixels}), skipping: {source_path}" - ) - return False - - aspect_ratio = width / height - if aspect_ratio < self.IMAGE_MIN_ASPECT_RATIO or aspect_ratio > self.IMAGE_MAX_ASPECT_RATIO: - logger.warning( - f"[MarkdownParser] Image aspect ratio out of range ({aspect_ratio:.4f}), " - f"skipping: {source_path}" - ) - return False - - return True - @staticmethod def _is_remote_uri(path: str) -> bool: """ @@ -1114,7 +1049,7 @@ async def _ingest_will_handle_image(self, link: str) -> bool: if resolved is not None: try: image_bytes = await asyncio.to_thread(resolved.read_bytes) - handled = await asyncio.to_thread(self._is_valid_image, image_bytes, resolved) + handled = await asyncio.to_thread(is_valid_image, image_bytes, resolved) except Exception: handled = False cache[link] = handled diff --git a/openviking/parse/parsers/pdf.py b/openviking/parse/parsers/pdf.py index 4d7acca204..45ac66854c 100644 --- a/openviking/parse/parsers/pdf.py +++ b/openviking/parse/parsers/pdf.py @@ -1,33 +1,17 @@ # Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. # SPDX-License-Identifier: AGPL-3.0 -""" -PDF parser for OpenViking. - -Unified parser that converts PDF to Markdown then parses the result. -Supports dual strategy: -- Local: pdfplumber for direct conversion -- Remote: MinerU API for advanced conversion - -This design simplifies PDF handling by delegating structure analysis -to the MarkdownParser after conversion. -""" +"""PDF parser using pdf-inspector for structure and pdfplumber for images.""" import asyncio import hashlib import io -import re import time -from collections import Counter, defaultdict +from dataclasses import dataclass, field from pathlib import Path from typing import Any, Dict, List, Optional, Union -from openviking.parse.base import ( - NodeType, - ParseResult, - ResourceNode, - create_parse_result, - lazy_import, -) +from openviking.parse.base import NodeType, ParseResult, ResourceNode, create_parse_result +from openviking.parse.image_validation import is_valid_image from openviking.parse.parsers.base_parser import BaseParser from openviking_cli.utils import get_logger from openviking_cli.utils.config.parser_config import PDFConfig @@ -35,50 +19,30 @@ logger = get_logger(__name__) -class PDFParser(BaseParser): - """ - PDF parser with dual conversion strategy. +@dataclass +class _PdfTextExtraction: + pages: dict[int, str] + warnings: list[str] + meta: dict[str, Any] - Converts PDF → Markdown → ParseResult using MarkdownParser. - When available, extracts PDF bookmarks/outlines and injects them as - markdown headings so MarkdownParser can build a hierarchical directory - structure instead of flat numbered files. - Strategies: - - "local": Use pdfplumber for text and table extraction - - "mineru": Use MinerU API for advanced PDF processing - - "auto": Try local first, fallback to MinerU if configured +@dataclass +class _PdfImageExtraction: + pages: dict[int, list[str]] = field(default_factory=dict) + warnings: list[str] = field(default_factory=list) + images_extracted: int = 0 + images_deduplicated: int = 0 - Examples: - >>> # Local parsing - >>> parser = PDFParser(PDFConfig(strategy="local")) - >>> result = await parser.parse("document.pdf") - >>> # Remote API parsing - >>> config = PDFConfig( - ... strategy="mineru", - ... mineru_endpoint="https://api.example.com/convert", - ... mineru_api_key="key" - ... ) - >>> parser = PDFParser(config) - >>> result = await parser.parse("document.pdf") - """ +class PDFParser(BaseParser): + """Convert a PDF to page-ordered Markdown and then build its context tree.""" def __init__(self, config: Optional[PDFConfig] = None): - """ - Initialize PDF parser. - - Args: - config: PDFConfig instance (defaults to auto strategy) - """ self.config = config or PDFConfig() self.config.validate() - - # Lazy import MarkdownParser to avoid circular imports self._markdown_parser = None def _get_markdown_parser(self): - """Lazy import and create MarkdownParser.""" if self._markdown_parser is None: from openviking.parse.parsers.markdown import MarkdownParser @@ -87,740 +51,228 @@ def _get_markdown_parser(self): @property def supported_extensions(self) -> List[str]: - """List of supported file extensions.""" return [".pdf"] async def parse(self, source: Union[str, Path], instruction: str = "", **kwargs) -> ParseResult: - """ - Parse PDF file. - - Args: - source: Path to PDF file - **kwargs: Additional options (resource_name/source_name for original filename) - - Returns: - ParseResult with document tree - - Raises: - FileNotFoundError: If PDF file doesn't exist - ValueError: If conversion fails with all strategies - """ - start_time = time.time() - pdf_path = Path(source) - - # Get resource name from kwargs, prefer original filename from upload + started = time.time() + path = Path(source) resource_name = kwargs.get("resource_name") or kwargs.get("source_name") + if not path.is_file(): + return self._failed_result(path, started, f"File not found: {path}") - if not pdf_path.exists(): - return create_parse_result( - root=ResourceNode(type=NodeType.ROOT), - source_path=str(pdf_path), - source_format="pdf", - parser_name="PDFParser", - parse_time=time.time() - start_time, - warnings=[f"File not found: {pdf_path}"], - ) + from openviking_cli.utils.storage import get_storage + storage = get_storage() try: - # Step 1: Convert PDF to Markdown - markdown_content, conversion_meta = await self._convert_to_markdown( - pdf_path, - resource_name=resource_name, + markdown, conversion_meta, conversion_warnings = await self._convert_to_markdown( + path, + storage=storage, + resource_name=resource_name or path.stem, ) - - # Step 2: Parse Markdown using MarkdownParser, pass through resource name - md_parser = self._get_markdown_parser() - from openviking_cli.utils.storage import get_storage - - storage = get_storage() - result = await md_parser.parse_content( - markdown_content, - source_path=str(pdf_path), - resource_name=resource_name, - source_name=resource_name, - base_dir=pdf_path.parent, - allowed_media_dirs=[storage.media_dir], - split_content=kwargs.get("split_content", True), + markdown_kwargs = dict(kwargs) + caller_media_dirs = list(markdown_kwargs.pop("allowed_media_dirs", None) or []) + caller_media_dirs.append(storage.media_dir) + result = await self._get_markdown_parser().parse_content( + markdown, + source_path=str(path), + instruction=instruction, + base_dir=path.parent, + allowed_media_dirs=caller_media_dirs, + **markdown_kwargs, ) - - # Step 3: Update metadata for PDF origin - result.source_format = "pdf" # Override markdown format + result.source_format = "pdf" result.parser_name = "PDFParser" - result.parser_version = "2.0" - result.parse_time = time.time() - start_time + result.parser_version = "3.0" + result.parse_time = time.time() - started + result.warnings.extend(conversion_warnings) result.meta.update(conversion_meta) - result.meta["pdf_strategy"] = self.config.strategy - result.meta["intermediate_markdown_length"] = len(markdown_content) - result.meta["intermediate_markdown_preview"] = markdown_content[:500] - + result.meta["intermediate_markdown_length"] = len(markdown) logger.info( - f"PDF parsed successfully: {pdf_path.name} " - f"({len(markdown_content)} chars markdown, " - f"{result.parse_time:.2f}s)" + f"PDF parsed successfully: {path.name} " + f"({len(markdown)} chars markdown, {result.parse_time:.2f}s)" ) - return result - - except Exception as e: - logger.error(f"Failed to parse PDF {pdf_path}: {e}") - return create_parse_result( - root=ResourceNode(type=NodeType.ROOT), - source_path=str(pdf_path), - source_format="pdf", - parser_name="PDFParser", - parse_time=time.time() - start_time, - warnings=[f"Failed to parse PDF: {e}"], - ) + except Exception as exc: + logger.error(f"Failed to parse PDF {path}: {exc}", exc_info=True) + return self._failed_result(path, started, f"Failed to parse PDF: {exc}") + + def _failed_result(self, path: Path, started: float, warning: str) -> ParseResult: + return create_parse_result( + root=ResourceNode(type=NodeType.ROOT), + source_path=str(path), + source_format="pdf", + parser_name="PDFParser", + parse_time=time.time() - started, + warnings=[warning], + ) async def _convert_to_markdown( self, - pdf_path: Path, - resource_name: Optional[str] = None, - ) -> tuple[str, Dict[str, Any]]: - """ - Convert PDF to Markdown using configured strategy. - - Args: - pdf_path: Path to PDF file - resource_name: Optional resource name for organizing saved images - - Returns: - Tuple of (markdown_content, metadata_dict) - - Raises: - ValueError: If all conversion strategies fail - """ - if self.config.strategy == "local": - return await self._convert_local(pdf_path, resource_name=resource_name) - - elif self.config.strategy == "mineru": - return await self._convert_mineru(pdf_path, resource_name=resource_name) - - elif self.config.strategy == "auto": - # Try local first - try: - return await self._convert_local(pdf_path, resource_name=resource_name) - except Exception as e: - logger.warning(f"Local conversion failed: {e}") - - # Fallback to MinerU if configured - if self.config.mineru_endpoint: - logger.info("Falling back to MinerU API") - return await self._convert_mineru(pdf_path, resource_name=resource_name) - else: - raise ValueError( - f"Local conversion failed and no MinerU endpoint configured: {e}" - ) - - else: - raise ValueError(f"Unknown strategy: {self.config.strategy}") - - async def _convert_local( - self, pdf_path: Path, storage=None, resource_name: Optional[str] = None - ) -> tuple[str, Dict[str, Any]]: - # pdfplumber / pdfminer 的解析与图片/表格提取通常是 CPU/IO 密集且为同步实现， - # 放到线程池中执行，避免阻塞事件循环。 - return await asyncio.to_thread(self._convert_local_sync, pdf_path, storage, resource_name) - - def _convert_local_sync( - self, pdf_path: Path, storage=None, resource_name: Optional[str] = None - ) -> tuple[str, Dict[str, Any]]: - """同步版：用 pdfplumber 将 PDF 转 Markdown。 - - 该方法会在 :meth:`_convert_local` 中通过 asyncio.to_thread 调用。 - """ - pdfplumber = lazy_import("pdfplumber") - - # Import storage utilities - if storage is None: - from openviking_cli.utils.storage import get_storage - - storage = get_storage() - - if resource_name is None: - resource_name = pdf_path.stem - - parts = [] - meta = { - "strategy": "local", - "library": "pdfplumber", - "pages_processed": 0, - "images_extracted": 0, - "images_deduplicated": 0, - "tables_extracted": 0, - "bookmarks_found": 0, - "bookmarks_resolved": 0, - "bookmarks_unresolved": 0, - "headings_found": 0, - "heading_source": "none", - } - - try: - with pdfplumber.open(str(pdf_path)) as pdf: - meta["total_pages"] = len(pdf.pages) - - # Extract structure (bookmarks → font fallback) - detection_mode = self.config.heading_detection - bookmarks = [] - raw_bookmarks = [] - heading_source = "none" - - if detection_mode in ("bookmarks", "auto"): - raw_bookmarks = self._extract_bookmarks(pdf) - meta["bookmarks_found"] = len(raw_bookmarks) - bookmarks = [bm for bm in raw_bookmarks if bm["page_num"] is not None] - meta["bookmarks_resolved"] = len(bookmarks) - meta["bookmarks_unresolved"] = len(raw_bookmarks) - len(bookmarks) - - if bookmarks: - heading_source = "bookmarks" - elif raw_bookmarks: - logger.info( - "Bookmark detection found %d entries but none resolved to pages; " - "ignoring bookmark headings", - len(raw_bookmarks), - ) - - if not bookmarks and detection_mode in ("font", "auto"): - bookmarks = self._detect_headings_by_font(pdf) - if bookmarks: - heading_source = "font_analysis" - - meta["headings_found"] = len(bookmarks) - meta["heading_source"] = heading_source - logger.info( - "Heading detection: source=%s, headings=%d, bookmarks=%d, resolved=%d, " - "unresolved=%d", - heading_source, - len(bookmarks), - meta["bookmarks_found"], - meta["bookmarks_resolved"], - meta["bookmarks_unresolved"], - ) - - # Group bookmarks by page_num - bookmarks_by_page = defaultdict(list) - for bm in bookmarks: - page = bm["page_num"] - if page is None: - continue - bookmarks_by_page[page].append(bm) - - for page_num, page in enumerate(pdf.pages, 1): - try: - # Inject headings before page text - page_bookmarks = bookmarks_by_page.get(page_num, []) - for bm in page_bookmarks: - heading_prefix = "#" * bm["level"] - parts.append(f"\n{heading_prefix} {bm['title']}\n") - - # Extract text - text = page.extract_text() - if text and text.strip(): - # Add page marker as HTML comment - parts.append(f"\n{text.strip()}") - meta["pages_processed"] += 1 - - # Extract tables - tables = page.extract_tables() - for table_idx, table in enumerate(tables or []): - if table and len(table) > 0: - md_table = self._format_table_markdown(table) - if md_table: - parts.append( - f"\n{md_table}" - ) - meta["tables_extracted"] += 1 - - # Extract images. - # - # A page can stack several image XObjects on the exact same - # spot — print-to-PDF producers routinely emit a background - # layer plus a content layer. Since extraction rasterises the - # page *region* rather than the XObject itself, every one of - # them renders to identical bytes. Skip the repeats: bbox - # first, which avoids the (expensive) render entirely, then a - # content hash as a backstop. Both sets are per-page, so a - # header logo repeated across pages is still kept once per - # page. - images = page.images - seen_boxes = set() - seen_digests = set() - for img_idx, img in enumerate(images or []): - try: - bbox_key = ( - round(img["x0"], 1), - round(img["top"], 1), - round(img["x1"], 1), - round(img["bottom"], 1), - ) - if bbox_key in seen_boxes: - meta["images_deduplicated"] += 1 - continue - seen_boxes.add(bbox_key) - - # Extract image using underlying PDF object - image_obj = self._extract_image_from_page(page, img) - if image_obj: - # Dedup only — md5 keeps this cheap, and the - # flag keeps it working on FIPS-locked hosts. - digest = hashlib.md5(image_obj, usedforsecurity=False).digest() - if digest in seen_digests: - meta["images_deduplicated"] += 1 - continue - seen_digests.add(digest) - - # Save image - filename = f"page{page_num}_img{img_idx + 1}" - image_path = storage.save_image( - resource_name, image_obj, filename=filename - ) - - # Generate path relative to the media root. - rel_path = image_path.relative_to(storage.media_dir) - parts.append( - f"\n" - f"![Page {page_num} Image {img_idx + 1}]({rel_path})" - ) - meta["images_extracted"] += 1 - except Exception as img_err: - logger.warning( - f"Failed to extract image {img_idx + 1} on page {page_num}: {img_err}" - ) - finally: - self._release_page_cache(page) - - if not parts: - logger.warning(f"No content extracted from {pdf_path}") - return "", meta - - markdown_content = "\n\n".join(parts) - logger.info( - f"Local conversion: {meta['pages_processed']}/{meta['total_pages']} pages, " - f"{meta['headings_found']} headings ({meta['heading_source']}, " - f"bookmarks={meta['bookmarks_found']}, " - f"resolved={meta['bookmarks_resolved']}), " - f"{meta['images_extracted']} images " - f"({meta['images_deduplicated']} duplicates skipped), " - f"{meta['tables_extracted']} tables → " - f"{len(markdown_content)} chars" + path: Path, + *, + storage: Any, + resource_name: str, + ) -> tuple[str, Dict[str, Any], list[str]]: + text_result, image_result = await asyncio.gather( + asyncio.to_thread(self._extract_text_sync, path), + asyncio.to_thread(self._extract_images_sync, path, storage, resource_name), + return_exceptions=True, + ) + if isinstance(text_result, BaseException): + raise text_result + if isinstance(image_result, BaseException): + image_result = _PdfImageExtraction( + warnings=[f"PDF image extraction failed: {image_result}"] ) - return markdown_content, meta - - except Exception as e: - logger.error(f"pdfplumber conversion failed: {e}") - raise + page_numbers = sorted(set(text_result.pages) | set(image_result.pages)) + parts: list[str] = [] + for page_number in page_numbers: + page_parts: list[str] = [] + page_markdown = text_result.pages.get(page_number, "").strip() + if page_markdown: + page_parts.append(page_markdown) + page_parts.extend(image_result.pages.get(page_number, [])) + if page_parts: + parts.append(f"\n\n" + "\n\n".join(page_parts)) + if not parts: + raise ValueError("No meaningful text or images extracted from PDF") + + meta = dict(text_result.meta) + meta.update( + { + "library": "pdf-inspector", + "image_library": "pdfplumber", + "images_extracted": image_result.images_extracted, + "images_deduplicated": image_result.images_deduplicated, + } + ) + return "\n\n".join(parts), meta, text_result.warnings + image_result.warnings @staticmethod - def _release_page_cache(page: Any) -> None: - """Release pdfplumber/pdfminer per-page caches when available.""" - close = getattr(page, "close", None) - if callable(close): - try: - close() - return - except Exception: - pass - - flush_cache = getattr(page, "flush_cache", None) - if callable(flush_cache): - try: - flush_cache() - except Exception: - pass - - def _extract_bookmarks(self, pdf) -> List[Dict[str, Any]]: - """Extract bookmark structure from PDF outlines. - - Returns: [{level: int, title: str, page_num: int(1-based)}] - """ - try: - if not hasattr(pdf, "doc") or not hasattr(pdf.doc, "get_outlines"): - return [] - - outlines = list(pdf.doc.get_outlines()) - if not outlines: - return [] - - page_ref_to_num = self._build_page_number_map(pdf) - - bookmarks = [] - for level, title, dest, _action, _se in outlines: - if not title or not title.strip(): - continue - - page_num = None - try: - if dest and len(dest) > 0: - page_num = self._resolve_bookmark_page( - dest[0], page_ref_to_num, len(pdf.pages) - ) - except Exception: - pass - - bookmarks.append( - { - "level": min(max(level, 1), 6), - "title": title.strip(), - "page_num": page_num, - } - ) - - return bookmarks - - except Exception as e: - logger.warning(f"Failed to extract bookmarks: {e}") - return [] - - def _build_page_number_map(self, pdf) -> Dict[int, int]: - """Build a lookup from PDF page object ids to 1-based page numbers. - - pdfminer outlines and link annotations reference page objects by object id. - In pdfplumber these ids are exposed as ``page.page_obj.pageid``; some mocks - or alternate inputs may still expose ``objid``, so we keep both. - """ - page_ref_to_num: Dict[int, int] = {} - for page_num, page in enumerate(pdf.pages, 1): - page_obj = getattr(page, "page_obj", None) - if page_obj is None: - continue - - for attr_name in ("pageid", "objid"): - ref_id = getattr(page_obj, attr_name, None) - if isinstance(ref_id, int): - page_ref_to_num.setdefault(ref_id, page_num) - - return page_ref_to_num - - def _resolve_bookmark_page( - self, page_ref: Any, page_ref_to_num: Dict[int, int], total_pages: int - ) -> Optional[int]: - """Resolve a bookmark destination to a 1-based page number.""" - ref_id = getattr(page_ref, "objid", None) - if isinstance(ref_id, int): - return page_ref_to_num.get(ref_id) - - if isinstance(page_ref, int): - # 0-based integer page index (common in many PDF producers) - candidate = page_ref + 1 - if 1 <= candidate <= total_pages: - return candidate - return None - - if hasattr(page_ref, "resolve"): - resolved = page_ref.resolve() - for attr_name in ("pageid", "objid"): - resolved_id = getattr(resolved, attr_name, None) - if isinstance(resolved_id, int): - return page_ref_to_num.get(resolved_id) + def _extract_text_sync(path: Path) -> _PdfTextExtraction: + import pdf_inspector - return None - - def _detect_headings_by_font(self, pdf) -> List[Dict[str, Any]]: - """Detect headings by font size analysis. - - Returns: [{level: int, title: str, page_num: int(1-based)}] - """ - try: - # Step 1: Sample font size distribution (every 5th page) - size_counter: Counter = Counter() - sample_pages = pdf.pages[::5] - for page in sample_pages: - try: - for char in page.chars: - if char["text"].strip(): - rounded = round(char["size"] * 2) / 2 - size_counter[rounded] += 1 - finally: - self._release_page_cache(page) - - if not size_counter: - return [] - - # Step 2: Determine body font size and heading font sizes - body_size = size_counter.most_common(1)[0][0] - min_delta = self.config.font_heading_min_delta - - heading_sizes = sorted( - [ - s - for s, count in size_counter.items() - if s >= body_size + min_delta and count < size_counter[body_size] * 0.5 - ], - reverse=True, - ) - - max_levels = self.config.max_heading_levels - heading_sizes = heading_sizes[:max_levels] - - if not heading_sizes: - logger.debug(f"Font analysis: body_size={body_size}pt, no heading sizes found") - return [] - - size_to_level = {s: i + 1 for i, s in enumerate(heading_sizes)} - logger.debug( - f"Font analysis: body_size={body_size}pt, " - f"heading_sizes={heading_sizes}, size_to_level={size_to_level}" - ) - - # Step 3: Extract heading text page by page - headings: List[Dict[str, Any]] = [] - - def flush_line(chars_to_flush: list, page_num: int) -> None: - if not chars_to_flush: - return - title = "".join(c["text"] for c in chars_to_flush).strip() - size = round(chars_to_flush[0]["size"] * 2) / 2 - - if len(title) < 2: - return - if len(title) > 100: - return - if title.isdigit(): - return - if re.match(r"^[\d\s.·…]+$", title): - return - - headings.append( - { - "level": size_to_level[size], - "title": title, - "page_num": page_num, - } - ) + extraction = pdf_inspector.extract_pages_markdown(str(path)) + pages = {page.page + 1: page.markdown or "" for page in extraction.pages} + reason_by_page = { + reason.page: list(reason.reasons) for reason in extraction.ocr_reasons_by_page + } + reasons = [ + {"page": page, "reasons": page_reasons} for page, page_reasons in reason_by_page.items() + ] + warnings = [ + f"PDF page {page} requires OCR: {', '.join(reason_by_page.get(page, ['unknown']))}" + for page in extraction.pages_needing_ocr + ] + meta = { + "total_pages": len(extraction.pages), + "pages_processed": sum(bool(markdown.strip()) for markdown in pages.values()), + "pages_needing_ocr": list(extraction.pages_needing_ocr), + "ocr_reasons_by_page": reasons, + "pages_with_tables": list(extraction.pages_with_tables), + "pages_with_columns": list(extraction.pages_with_columns), + "is_complex_layout": extraction.is_complex, + } + return _PdfTextExtraction(pages=pages, warnings=warnings, meta=meta) - for page in pdf.pages: + def _extract_images_sync( + self, + path: Path, + storage: Any, + resource_name: str, + ) -> _PdfImageExtraction: + import pdfplumber + + result = _PdfImageExtraction() + with pdfplumber.open(str(path)) as pdf: + for page_number, page in enumerate(pdf.pages, 1): try: - page_num = page.page_number + 1 - chars = sorted(page.chars, key=lambda c: (c["top"], c["x0"])) - - current_line_chars: list = [] - current_top = None - - for char in chars: - # Performance: headings won't appear in bottom 70% of page - if char["top"] > page.height * 0.3: - flush_line(current_line_chars, page_num) - current_line_chars = [] - break - - rounded_size = round(char["size"] * 2) / 2 - if rounded_size not in size_to_level: - flush_line(current_line_chars, page_num) - current_line_chars = [] - current_top = None - continue - - # Same line check (top offset < 2pt) - if current_top is not None and abs(char["top"] - current_top) > 2: - flush_line(current_line_chars, page_num) - current_line_chars = [] - - current_line_chars.append(char) - current_top = char["top"] - - flush_line(current_line_chars, page_num) + seen_boxes: set[tuple[float, float, float, float]] = set() + seen_digests: set[bytes] = set() + page_images: list[str] = [] + for image_index, image in enumerate(page.images or [], 1): + try: + box = ( + round(image["x0"], 1), + round(image["top"], 1), + round(image["x1"], 1), + round(image["bottom"], 1), + ) + if box in seen_boxes: + result.images_deduplicated += 1 + continue + seen_boxes.add(box) + image_bytes = self._extract_image_from_page(page, image) + if image_bytes is None: + continue + filename = f"page{page_number}_img{image_index}" + if not is_valid_image(image_bytes, Path(f"{filename}.png")): + continue + digest = hashlib.md5(image_bytes, usedforsecurity=False).digest() + if digest in seen_digests: + result.images_deduplicated += 1 + continue + seen_digests.add(digest) + image_path = storage.save_image( + resource_name, + image_bytes, + filename=filename, + extension=".png", + ) + relative = image_path.relative_to(storage.media_dir).as_posix() + page_images.append( + f"![Page {page_number} Image {image_index}]({relative})" + ) + result.images_extracted += 1 + except Exception as exc: + warning = ( + f"Failed to extract image {image_index} on page " + f"{page_number}: {exc}" + ) + logger.warning(warning) + result.warnings.append(warning) + if page_images: + result.pages[page_number] = page_images finally: self._release_page_cache(page) - - # Step 4: Deduplicate - filter headers appearing on >30% of pages - title_page_count: Counter = Counter(h["title"] for h in headings) - total_pages = len(pdf.pages) - header_titles = {t for t, c in title_page_count.items() if c > total_pages * 0.3} - headings = [h for h in headings if h["title"] not in header_titles] - - logger.debug( - f"Font heading detection: {len(headings)} headings found " - f"(filtered {len(header_titles)} header titles)" - ) - return headings - - except Exception as e: - logger.warning(f"Failed to detect headings by font: {e}") - return [] - - def _extract_image_from_page(self, page, img_info: dict) -> Optional[bytes]: - """ - Extract a PDF image as valid PNG bytes. - - Renders the image's bounding box on the page to a raster PNG via - pdfplumber's ``crop().to_image()`` instead of returning the raw decoded - XObject stream (which is not a valid image file and cannot be opened). - - Args: - page: pdfplumber page object - img_info: Image metadata from page.images - - Returns: - PNG-encoded image bytes or None if extraction fails - """ - try: - # pdfplumber coordinates: ``top`` is measured from the top of the page. - bbox = ( - max(0, img_info["x0"]), - max(0, img_info["top"]), - min(page.width, img_info["x1"]), - min(page.height, img_info["bottom"]), - ) - - # Skip degenerate / zero-area boxes that cannot be cropped. - if bbox[2] <= bbox[0] or bbox[3] <= bbox[1]: - return None - - cropped = page.crop(bbox) - page_image = cropped.to_image(resolution=self.config.image_resolution) - - buffer = io.BytesIO() - page_image.save(buffer, format="PNG") - return buffer.getvalue() - - except Exception as e: - logger.debug(f"Image extraction error: {e}") + return result + + def _extract_image_from_page(self, page: Any, image: dict[str, Any]) -> Optional[bytes]: + bbox = ( + max(0, image["x0"]), + max(0, image["top"]), + min(page.width, image["x1"]), + min(page.height, image["bottom"]), + ) + if bbox[2] <= bbox[0] or bbox[3] <= bbox[1]: return None + cropped = page.crop(bbox) + rendered = cropped.to_image(resolution=self.config.image_resolution) + buffer = io.BytesIO() + rendered.save(buffer, format="PNG") + return buffer.getvalue() - async def _convert_mineru( - self, - pdf_path: Path, - resource_name: Optional[str] = None, - ) -> tuple[str, Dict[str, Any]]: - """ - Convert PDF to Markdown using MinerU API. - - Args: - pdf_path: Path to PDF file - resource_name: Optional resource name (unused in MinerU conversion) - - Returns: - Tuple of (markdown_content, metadata) - - Raises: - ImportError: If httpx not installed - Exception: If API call fails - """ - httpx = lazy_import("httpx") - - if not self.config.mineru_endpoint: - raise ValueError("MinerU endpoint not configured") - - meta = { - "strategy": "mineru", - "endpoint": self.config.mineru_endpoint, - "api_version": None, - } - + @staticmethod + def _release_page_cache(page: Any) -> None: try: - async with httpx.AsyncClient(timeout=self.config.mineru_timeout) as client: - # Prepare file upload - with open(pdf_path, "rb") as f: - files = {"file": (pdf_path.name, f, "application/pdf")} - - # Prepare headers - headers = {} - if self.config.mineru_api_key: - headers["Authorization"] = f"Bearer {self.config.mineru_api_key}" - - # Prepare request params - params = self.config.mineru_params or {} - - # Make API request - logger.info(f"Calling MinerU API: {self.config.mineru_endpoint}") - response = await client.post( - self.config.mineru_endpoint, - files=files, - headers=headers, - params=params, - ) - response.raise_for_status() - - # Parse response - result = response.json() - markdown_content = result.get("markdown", "") - - # Extract metadata from response - meta["api_version"] = result.get("version") - meta["processing_time"] = result.get("processing_time") - meta["total_pages"] = result.get("total_pages") - - if not markdown_content: - logger.warning(f"MinerU returned empty content for {pdf_path}") - - logger.info( - f"MinerU conversion: {meta.get('total_pages', '?')} pages → " - f"{len(markdown_content)} chars" - ) - - return markdown_content, meta - - except Exception as e: - logger.error(f"MinerU API call failed: {e}") - raise - - def _format_table_markdown(self, table: List[List[Optional[str]]]) -> str: - """ - Convert table data to Markdown table format. - - Args: - table: 2D array of table cells - - Returns: - Markdown table string - - Examples: - >>> table = [["Name", "Age"], ["Alice", "30"], ["Bob", "25"]] - >>> print(parser._format_table_markdown(table)) - | Name | Age | - | --- | --- | - | Alice | 30 | - | Bob | 25 | - """ - if not table or not table[0]: - return "" - - # Clean cells and handle None values - def clean_cell(cell): - if cell is None: - return "" - return str(cell).strip().replace("|", "\\|") # Escape pipe characters - - lines = [] - - # Header row - header = table[0] - header_cells = [clean_cell(cell) for cell in header] - lines.append("| " + " | ".join(header_cells) + " |") - - # Separator row - separator = ["---"] * len(header) - lines.append("| " + " | ".join(separator) + " |") - - # Data rows - for row in table[1:]: - # Pad row to match header length - padded_row = row + [None] * (len(header) - len(row)) - cells = [clean_cell(cell) for cell in padded_row[: len(header)]] - lines.append("| " + " | ".join(cells) + " |") - - return "\n".join(lines) + close = getattr(page, "close", None) + if callable(close): + close() + return + flush_cache = getattr(page, "flush_cache", None) + if callable(flush_cache): + flush_cache() + except Exception as exc: + logger.debug(f"Failed to release pdfplumber page cache: {exc}") async def parse_content( - self, content: str, source_path: Optional[str] = None, instruction: str = "", **kwargs + self, + content: str, + source_path: Optional[str] = None, + instruction: str = "", + **kwargs, ) -> ParseResult: - """ - Parse PDF content string. - - Note: This method is not recommended for PDFParser as it requires - file path for conversion tools. Use parse() with file path instead. - - Args: - content: PDF content (not supported) - source_path: Optional source path - **kwargs: Additional options - - Raises: - NotImplementedError: PDFParser requires file path - """ raise NotImplementedError( - "PDFParser does not support parsing content strings. " - "Use parse() with a file path instead." + "PDFParser requires a PDF file path; string content is not supported" ) diff --git a/openviking/parse/parsers/powerpoint.py b/openviking/parse/parsers/powerpoint.py deleted file mode 100644 index 9ef1839969..0000000000 --- a/openviking/parse/parsers/powerpoint.py +++ /dev/null @@ -1,150 +0,0 @@ -# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. -# SPDX-License-Identifier: AGPL-3.0 -""" -PowerPoint (.pptx) parser for OpenViking. - -Converts PowerPoint presentations to Markdown then parses using MarkdownParser. -Inspired by microsoft/markitdown approach. -""" - -import asyncio -from pathlib import Path -from typing import List, Optional, Union - -from openviking.parse.base import ParseResult -from openviking.parse.parsers.base_parser import BaseParser -from openviking_cli.utils.config.parser_config import ParserConfig -from openviking_cli.utils.logger import get_logger - -logger = get_logger(__name__) - - -class PowerPointParser(BaseParser): - """ - PowerPoint presentation parser for OpenViking. - - Supports: .pptx - - Converts PowerPoint presentations to Markdown using python-pptx, - then delegates to MarkdownParser for tree structure creation. - """ - - def __init__(self, config: Optional[ParserConfig] = None, extract_notes: bool = False): - """ - Initialize PowerPoint parser. - - Args: - config: Parser configuration - extract_notes: Whether to extract speaker notes - """ - from openviking.parse.parsers.markdown import MarkdownParser - - self._md_parser = MarkdownParser(config=config) - self.config = config or ParserConfig() - self.extract_notes = extract_notes - - @property - def supported_extensions(self) -> List[str]: - return [".pptx"] - - async def parse(self, source: Union[str, Path], instruction: str = "", **kwargs) -> ParseResult: - """Parse PowerPoint presentation from file path.""" - path = Path(source) - - if path.exists(): - import pptx - - markdown_content = await asyncio.to_thread(self._convert_to_markdown, path, pptx) - result = await self._md_parser.parse_content( - markdown_content, source_path=str(path), instruction=instruction, **kwargs - ) - else: - result = await self._md_parser.parse_content( - str(source), instruction=instruction, **kwargs - ) - result.source_format = "pptx" - result.parser_name = "PowerPointParser" - return result - - async def parse_content( - self, content: str, source_path: Optional[str] = None, instruction: str = "", **kwargs - ) -> ParseResult: - """Parse content - delegates to MarkdownParser.""" - result = await self._md_parser.parse_content(content, source_path, **kwargs) - result.source_format = "pptx" - result.parser_name = "PowerPointParser" - return result - - def _convert_to_markdown(self, path: Path, pptx) -> str: - """Convert PowerPoint presentation to Markdown string.""" - prs = pptx.Presentation(path) - markdown_parts = [] - slide_count = len(prs.slides) - - for idx, slide in enumerate(prs.slides, 1): - slide_parts = [] - slide_parts.append(f"## Slide {idx}/{slide_count}") - - title = self._extract_slide_title(slide) - if title: - slide_parts.append(f"### {title}") - - content = self._extract_slide_content(slide) - if content: - slide_parts.append(content) - - if self.extract_notes and slide.has_notes_slide: - notes = slide.notes_slide.notes_text_frame.text.strip() - if notes: - slide_parts.append(f"**Notes:** {notes}") - - markdown_parts.append("\n\n".join(slide_parts)) - - return "\n\n---\n\n".join(markdown_parts) - - def _extract_slide_title(self, slide) -> str: - """Extract title from a slide.""" - from pptx.enum.shapes import PP_PLACEHOLDER - - for shape in slide.shapes: - if shape.is_placeholder: - ph_type = shape.placeholder_format.type - if ph_type in (PP_PLACEHOLDER.TITLE, PP_PLACEHOLDER.CENTER_TITLE): - return shape.text.strip() - return "" - - def _extract_slide_content(self, slide) -> str: - """Extract content from slide shapes.""" - from pptx.enum.shapes import PP_PLACEHOLDER - - content_parts = [] - - for shape in slide.shapes: - if shape.is_placeholder: - ph_type = shape.placeholder_format.type - if ph_type in (PP_PLACEHOLDER.TITLE, PP_PLACEHOLDER.CENTER_TITLE): - continue - - if hasattr(shape, "text") and shape.text.strip(): - if shape.has_table: - content_parts.append(self._convert_table(shape.table)) - else: - text = shape.text.strip() - if text: - content_parts.append(text) - - return "\n\n".join(content_parts) - - def _convert_table(self, table) -> str: - """Convert PowerPoint table to markdown format.""" - if not table.rows: - return "" - - rows = [] - for row in table.rows: - row_data = [cell.text.strip() for cell in row.cells] - rows.append(row_data) - - from openviking.parse.base import format_table_to_markdown - - return format_table_to_markdown(rows, has_header=True) diff --git a/openviking/parse/parsers/word.py b/openviking/parse/parsers/word.py deleted file mode 100644 index 0a76f110ae..0000000000 --- a/openviking/parse/parsers/word.py +++ /dev/null @@ -1,231 +0,0 @@ -# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. -# SPDX-License-Identifier: AGPL-3.0 -""" -Word document (.docx) parser for OpenViking. - -Converts Word documents to Markdown then parses using MarkdownParser. -Inspired by microsoft/markitdown approach. -""" - -import asyncio -from pathlib import Path -from typing import List, Optional, Union - -from openviking.parse.base import ParseResult -from openviking.parse.parsers.base_parser import BaseParser -from openviking_cli.utils.config.parser_config import ParserConfig -from openviking_cli.utils.logger import get_logger - -logger = get_logger(__name__) - - -class WordParser(BaseParser): - """ - Word document parser for OpenViking. - - Supports: .docx - - Converts Word documents to Markdown using python-docx, - then delegates to MarkdownParser for tree structure creation. - """ - - def __init__(self, config: Optional[ParserConfig] = None): - """Initialize Word parser.""" - from openviking.parse.parsers.markdown import MarkdownParser - - self._md_parser = MarkdownParser(config=config) - self.config = config or ParserConfig() - - @property - def supported_extensions(self) -> List[str]: - return [".docx"] - - async def parse(self, source: Union[str, Path], instruction: str = "", **kwargs) -> ParseResult: - """Parse Word document from file path.""" - path = Path(source) - - if path.exists(): - import docx - - from openviking_cli.utils.storage import get_storage - - storage = get_storage() - resource_name = kwargs.get("resource_name") or kwargs.get("source_name") or path.stem - - markdown_content = await asyncio.to_thread( - self._convert_to_markdown, path, docx, resource_name, storage - ) - result = await self._md_parser.parse_content( - markdown_content, - source_path=str(path), - # Forward the original upload name explicitly (mirrors pdf.py) so - # the resource is named after it, not the temp upload path. Pass - # the raw kwargs values rather than the path.stem-backed local - # resource_name, leaving MarkdownParser's naming logic unchanged. - resource_name=kwargs.get("resource_name"), - source_name=kwargs.get("source_name"), - instruction=instruction, - base_dir=path.parent, - # docx images are extracted into storage.media_dir, so (like pdf) - # that is the only derived-media root needed. - allowed_media_dirs=[storage.media_dir], - split_content=kwargs.get("split_content", True), - ) - else: - result = await self._md_parser.parse_content( - str(source), - instruction=instruction, - resource_name=kwargs.get("resource_name"), - source_name=kwargs.get("source_name"), - split_content=kwargs.get("split_content", True), - ) - result.source_format = "docx" - result.parser_name = "WordParser" - return result - - async def parse_content( - self, content: str, source_path: Optional[str] = None, instruction: str = "", **kwargs - ) -> ParseResult: - """Parse content - delegates to MarkdownParser.""" - result = await self._md_parser.parse_content(content, source_path, **kwargs) - result.source_format = "docx" - result.parser_name = "WordParser" - return result - - def _convert_to_markdown(self, path: Path, docx, resource_name=None, storage=None) -> str: - """Convert Word document to Markdown string. - - Iterates the document body in order so that tables appear in their - original position rather than being appended at the end. Embedded - images are extracted to local storage and referenced inline so the - MarkdownParser can ingest them into VikingFS. - """ - doc = docx.Document(path) - markdown_parts = [] - - # Map XML table elements to python-docx Table objects for O(1) lookup - table_by_element = {table._tbl: table for table in doc.tables} - - # Track extracted images to deduplicate and number sequentially - image_counter = [0] - - # Walk the document body in order to preserve table positions - from docx.oxml.ns import qn - - for child in doc.element.body: - if child.tag == qn("w:p"): - # It's a paragraph - from docx.text.paragraph import Paragraph - - paragraph = Paragraph(child, doc) - - # Extract any embedded images in this paragraph first so they - # keep their original document position. - image_md = self._convert_paragraph_images( - paragraph, doc, qn, resource_name, storage, image_counter - ) - if image_md: - markdown_parts.append(image_md) - - if not paragraph.text.strip(): - continue - - style_name = paragraph.style.name if paragraph.style else "Normal" - - if style_name.startswith("Heading"): - level = self._extract_heading_level(style_name) - markdown_parts.append(f"{'#' * level} {paragraph.text}") - else: - text = self._convert_formatted_text(paragraph) - markdown_parts.append(text) - - elif child.tag == qn("w:tbl"): - # It's a table - if child in table_by_element: - markdown_parts.append(self._convert_table(table_by_element[child])) - - return "\n\n".join(markdown_parts) - - def _convert_paragraph_images( - self, paragraph, doc, qn, resource_name, storage, image_counter - ) -> str: - """Extract embedded images from a paragraph and return markdown references. - - Images in .docx are stored as relationship parts (``word/media/*``) - referenced by ``r:embed`` ids inside ``w:drawing`` elements. We pull the - binary blob for each and persist it via the shared storage helper so the - MarkdownParser's image ingestion can pick it up. - """ - if storage is None: - return "" - - parts = [] - # Each inside the paragraph references an image part - for blip in paragraph._p.findall(".//" + qn("a:blip")): - rid = blip.get(qn("r:embed")) - if not rid: - continue - try: - image_part = doc.part.related_parts[rid] - image_bytes = image_part.blob - except Exception as e: - logger.warning(f"[WordParser] Failed to read embedded image {rid}: {e}") - continue - - try: - extension = Path(image_part.partname).suffix or ".png" - image_counter[0] += 1 - filename = f"image{image_counter[0]}" - image_path = storage.save_image( - resource_name, image_bytes, filename=filename, extension=extension - ) - # Reference relative to the media root so the MarkdownParser can - # resolve it within the resource's media lifecycle (never cwd). - rel_path = image_path.relative_to(storage.media_dir) - parts.append(f"![{filename}]({rel_path})") - except Exception as e: - logger.warning(f"[WordParser] Failed to save embedded image {rid}: {e}") - - return "\n\n".join(parts) - - def _extract_heading_level(self, style_name: str) -> int: - """Extract heading level from style name.""" - try: - if "Heading" in style_name: - parts = style_name.split() - for part in parts: - if part.isdigit(): - return min(int(part), 6) - except Exception: - pass - return 1 - - def _convert_formatted_text(self, paragraph) -> str: - """Convert paragraph with formatting to markdown.""" - text_parts = [] - for run in paragraph.runs: - text = run.text - if not text: - continue - if run.bold: - text = f"**{text}**" - if run.italic: - text = f"*{text}*" - if run.underline: - text = f"{text}" - text_parts.append(text) - return "".join(text_parts) - - def _convert_table(self, table) -> str: - """Convert Word table to markdown format.""" - if not table.rows: - return "" - - rows = [] - for row in table.rows: - row_data = [cell.text.strip() for cell in row.cells] - rows.append(row_data) - - from openviking.parse.base import format_table_to_markdown - - return format_table_to_markdown(rows, has_header=True) diff --git a/openviking/parse/registry.py b/openviking/parse/registry.py index dc0140562d..5db8868c86 100644 --- a/openviking/parse/registry.py +++ b/openviking/parse/registry.py @@ -10,24 +10,18 @@ from typing import Dict, List, Optional, Union from openviking.parse.base import ParseResult +from openviking.parse.parsers.anydoc import AnyDocParser from openviking.parse.parsers.base_parser import BaseParser from openviking.parse.parsers.constants import TYPESCRIPT_MPEG_TS_EXTENSION from openviking.parse.parsers.directory import DirectoryParser -from openviking.parse.parsers.epub import EPubParser -from openviking.parse.parsers.excel import ExcelParser # Import will be handled dynamically to avoid dependency issues from openviking.parse.parsers.html import HTMLParser - -# Import markitdown-inspired parsers -from openviking.parse.parsers.legacy_doc import LegacyDocParser from openviking.parse.parsers.markdown import MarkdownParser from openviking.parse.parsers.media import AudioParser, ImageParser, VideoParser from openviking.parse.parsers.media.utils import is_mpeg_ts, read_mpeg_ts_probe from openviking.parse.parsers.pdf import PDFParser -from openviking.parse.parsers.powerpoint import PowerPointParser from openviking.parse.parsers.text import TextParser -from openviking.parse.parsers.word import WordParser from openviking.parse.parsers.zip_parser import ZipParser from openviking_cli.utils.config.parser_config import ParserConfig @@ -59,14 +53,7 @@ def __init__( self._register("pdf", PDFParser(config=self._parser_configs.get("pdf"))) self._register("html", HTMLParser(config=self._parser_configs.get("html"))) - # Register markitdown-inspired parsers (built-in) - self._register("word", WordParser(config=self._parser_configs.get("word"))) - self._register("legacy_doc", LegacyDocParser(config=self._parser_configs.get("legacy_doc"))) - self._register( - "powerpoint", PowerPointParser(config=self._parser_configs.get("powerpoint")) - ) - self._register("excel", ExcelParser(config=self._parser_configs.get("excel"))) - self._register("epub", EPubParser(config=self._parser_configs.get("epub"))) + self._register("anydoc", AnyDocParser(config=self._parser_configs.get("anydoc"))) self._register("zip", ZipParser()) self._register("directory", DirectoryParser()) @@ -176,15 +163,7 @@ def get_registry() -> ParserRegistry: "markdown": config.markdown, "pdf": config.pdf, "html": config.html, - "word": config.markdown, - "legacy_doc": config.markdown, - "powerpoint": config.markdown, - # Excel had no dedicated config section and reused - # ``config.markdown``. Keep unset sectioning fields following - # Markdown so existing deployments keep their node structure and - # stable URIs after this upgrade. - "excel": config.excel.with_sectioning_defaults_from(config.markdown), - "epub": config.markdown, + "anydoc": config.markdown, "image": config.image, } except Exception: diff --git a/openviking_cli/utils/config/__init__.py b/openviking_cli/utils/config/__init__.py index b84173610a..3284689ee3 100644 --- a/openviking_cli/utils/config/__init__.py +++ b/openviking_cli/utils/config/__init__.py @@ -67,7 +67,6 @@ PARSER_CONFIG_REGISTRY, AudioConfig, CodeConfig, - ExcelConfig, HTMLConfig, ImageConfig, MarkdownConfig, @@ -147,7 +146,6 @@ "AudioConfig", "VideoConfig", "MarkdownConfig", - "ExcelConfig", "HTMLConfig", "TextConfig", "get_parser_config", diff --git a/openviking_cli/utils/config/open_viking_config.py b/openviking_cli/utils/config/open_viking_config.py index c7fbb24519..e7e467f573 100644 --- a/openviking_cli/utils/config/open_viking_config.py +++ b/openviking_cli/utils/config/open_viking_config.py @@ -31,7 +31,6 @@ AudioConfig, CodeConfig, DirectoryConfig, - ExcelConfig, FeishuConfig, HTMLConfig, ImageConfig, @@ -205,14 +204,6 @@ class OpenVikingConfig(BaseModel): default_factory=MarkdownConfig, description="Markdown parsing configuration" ) - excel: ExcelConfig = Field( - # from_dict on an empty mapping, not the bare constructor: an absent - # parsers.excel section must record that no key was set, so sectioning - # still follows parsers.markdown for deployments predating this section. - default_factory=lambda: ExcelConfig.from_dict({}), - description="Excel parsing configuration", - ) - html: HTMLConfig = Field(default_factory=HTMLConfig, description="HTML parsing configuration") text: TextConfig = Field(default_factory=TextConfig, description="Text parsing configuration") @@ -401,7 +392,6 @@ def from_dict(cls, config: Dict[str, Any]) -> "OpenVikingConfig": "audio", "video", "markdown", - "excel", "html", "text", "directory", diff --git a/openviking_cli/utils/config/parser_config.py b/openviking_cli/utils/config/parser_config.py index 83fd271c20..74f2abf41d 100644 --- a/openviking_cli/utils/config/parser_config.py +++ b/openviking_cli/utils/config/parser_config.py @@ -8,9 +8,9 @@ and can be loaded from ov.conf files. """ -from dataclasses import dataclass, replace +from dataclasses import dataclass from pathlib import Path -from typing import Any, Dict, Iterable, Optional, Union +from typing import Any, Dict, Optional, Union from openviking_cli.utils.logger import get_logger @@ -137,34 +137,7 @@ def to_dict(self) -> Dict[str, Any]: @dataclass class PDFConfig(ParserConfig): - """ - Configuration for PDF parsing. - - Supports three strategies: - - "local": Use pdfplumber for local PDF→Markdown conversion - - "mineru": Use MinerU API for remote PDF→Markdown conversion - - "auto": Try local first, fallback to MinerU if available - - Attributes: - strategy: Parsing strategy ("local" | "mineru" | "auto") - mineru_endpoint: MinerU API endpoint URL - mineru_api_key: MinerU API authentication key - mineru_timeout: MinerU request timeout in seconds - mineru_params: Additional MinerU API parameters - """ - - strategy: str = "auto" # "local" | "mineru" | "auto" - - # MinerU API configuration - mineru_endpoint: Optional[str] = None # API endpoint URL - mineru_api_key: Optional[str] = None # API authentication key - mineru_timeout: float = 300.0 # Request timeout in seconds (5 minutes) - mineru_params: Optional[dict] = None # Additional API parameters - - # Heading detection configuration - heading_detection: str = "auto" # "bookmarks" | "font" | "auto" | "none" - font_heading_min_delta: float = 1.5 # Minimum font size delta from body text (pt) - max_heading_levels: int = 4 # Maximum heading levels for font analysis + """Configuration for local PDF parsing.""" # Image extraction configuration image_resolution: int = 300 # Rendering DPI for extracted image regions @@ -179,24 +152,8 @@ def validate(self) -> None: # Validate base class fields super().validate() - # Validate PDF-specific fields - if self.strategy not in ("local", "mineru", "auto"): - raise ValueError( - f"Invalid strategy '{self.strategy}'. Must be 'local', 'mineru', or 'auto'" - ) - - if self.strategy == "mineru": - if not self.mineru_endpoint: - raise ValueError("mineru_endpoint is required when strategy='mineru'") - - if self.mineru_timeout <= 0: - raise ValueError("mineru_timeout must be positive") - - if self.heading_detection not in ("bookmarks", "font", "auto", "none"): - raise ValueError(f"Invalid heading_detection: {self.heading_detection}") - - if self.font_heading_min_delta <= 0: - raise ValueError("font_heading_min_delta must be positive") + if self.image_resolution <= 0: + raise ValueError("image_resolution must be positive") @dataclass @@ -467,95 +424,6 @@ def validate(self) -> None: raise ValueError("max_heading_depth must be at least 1") -@dataclass -class ExcelConfig(ParserConfig): - """ - Configuration for Excel parsing. - - Attributes: - enable_process_pool: Offload Excel→Markdown conversion and layout - planning to a ProcessPoolExecutor (default off). - process_pool_workers: Max worker processes when the pool is enabled. - """ - - enable_process_pool: bool = False - process_pool_workers: int = 2 - - # Excel is converted to Markdown and then sectioned by MarkdownParser, so - # these fields decide the resulting node structure and stable URIs. - _SECTIONING_FIELDS = ( - "max_content_length", - "encoding", - "max_section_size", - "section_size_flexibility", - "max_section_chars", - ) - - # Names of keys a config source actually provided. Tracked as a plain - # instance attribute rather than a dataclass field so it never appears in - # asdict/model_dump output, cannot be injected from a config file, and does - # not affect equality. Absent means "provenance unknown". - _EXPLICIT_ATTR = "_openviking_explicit_keys" - - @classmethod - def from_dict(cls, data: Dict[str, Any]) -> "ExcelConfig": - """Build the config while remembering which keys were actually present. - - ``with_sectioning_defaults_from`` needs to tell "the user wrote this - value" from "the key was absent". Comparing against class defaults - cannot do that, so record the provided keys here instead. - """ - config = super().from_dict(data) - return config.with_explicit_keys(data) - - def with_explicit_keys(self, names: Iterable[str]) -> "ExcelConfig": - """Return this config marked as having ``names`` explicitly configured.""" - object.__setattr__(self, self._EXPLICIT_ATTR, frozenset(names)) - return self - - @property - def explicit_keys(self) -> Optional[frozenset]: - """Keys a config source provided, or ``None`` when unknown.""" - return getattr(self, self._EXPLICIT_ATTR, None) - - def with_sectioning_defaults_from(self, markdown: "ParserConfig") -> "ExcelConfig": - """Inherit sectioning fields that ``parsers.excel`` did not set. - - Excel used to be registered with ``config.markdown`` directly, so a - deployment that tuned ``parsers.markdown`` also tuned Excel imports. - Introducing a dedicated ``parsers.excel`` section must not silently - change that node structure, so a sectioning field absent from - ``parsers.excel`` keeps following Markdown. Explicit ``parsers.excel`` - values always win, including one that happens to equal the class - default. - - Configs built without ``from_dict`` carry no key information; those are - treated as fully explicit so a hand-constructed ``ExcelConfig`` is never - silently rewritten. - """ - if markdown is None: - return self - - explicit = self.explicit_keys - if explicit is None: - return self - - overrides = { - name: getattr(markdown, name) - for name in self._SECTIONING_FIELDS - if hasattr(markdown, name) and name not in explicit - } - if not overrides: - return self - return replace(self, **overrides).with_explicit_keys(explicit) - - def validate(self) -> None: - """Validate Excel-specific configuration.""" - super().validate() - if self.process_pool_workers < 1: - raise ValueError("process_pool_workers must be at least 1") - - @dataclass class HTMLConfig(ParserConfig): """ @@ -782,7 +650,6 @@ def __post_init__(self): "audio": AudioConfig, "video": VideoConfig, "markdown": MarkdownConfig, - "excel": ExcelConfig, "html": HTMLConfig, "text": TextConfig, "directory": DirectoryConfig, @@ -840,7 +707,7 @@ def load_parser_configs_from_dict(config_dict: Dict[str, Any]) -> Dict[str, Pars Examples: >>> configs = load_parser_configs_from_dict({ - ... "pdf": {"strategy": "auto"}, + ... "pdf": {"image_resolution": 300}, ... "code": {"github_raw_domain": "raw.githubusercontent.com"} ... }) >>> pdf_config = configs["pdf"] diff --git a/pyproject.toml b/pyproject.toml index defbab250d..c8a9a35bc0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,8 @@ dependencies = [ "typing-extensions>=4.5.0", "pyyaml>=6.0", "httpx>=0.25.0", + "firecrawl-anydoc>=0.1.8,<0.2", + "pdf-inspector>=1.14.1,<2", "pdfplumber>=0.10.0", "scrapy>=2.11.0", "trafilatura>=1.12.0", @@ -43,12 +45,6 @@ dependencies = [ "openai>=1.0.0", "requests>=2.33.0", "charset-normalizer>=3.4,<4", - "python-docx>=1.0.0", - "olefile>=0.47", - "xlrd>=2.0.1", - "python-pptx>=1.0.0", - "openpyxl>=3.0.0", - "ebooklib>=0.18.0", "json-repair>=0.25.0", "apscheduler>=3.11.0", "volcengine>=1.0.216", @@ -60,7 +56,6 @@ dependencies = [ "tabulate>=0.9.0", "urllib3>=2.7.0", "protobuf>=6.33.5", - "pdfminer-six>=20251230", "typer>=0.12.0", "litellm>=1.83.7,<1.91.2", "python-multipart>=0.0.31", @@ -104,6 +99,10 @@ test = [ "pandas>=2.0.0", "diff-match-patch>=20200713", "hvac>=2.0.0", + "python-docx>=1.0.0", + "python-pptx>=1.0.0", + "openpyxl>=3.0.0", + "ebooklib>=0.18.0", ] auth = [ "python-jose[cryptography]>=3.3.0", diff --git a/tests/parse/test_add_directory.py b/tests/parse/test_add_directory.py index 70269b62d3..7482baccc5 100644 --- a/tests/parse/test_add_directory.py +++ b/tests/parse/test_add_directory.py @@ -430,143 +430,39 @@ async def test_txt_file_goes_through_parser(self, tmp_path: Path, parser, fake_f assert len(fake_fs.files) > 0 @pytest.mark.asyncio - async def test_docx_file_goes_through_parser(self, tmp_path: Path, parser, fake_fs) -> None: - """Word (.docx) files should be processed by WordParser.parse().""" - (tmp_path / "report.docx").write_bytes(b"PK\x03\x04") - - mock_temp = fake_fs.create_temp_uri() - doc_dir = f"{mock_temp}/report" - await fake_fs.mkdir(mock_temp) - await fake_fs.mkdir(doc_dir) - await fake_fs.write_file(f"{doc_dir}/report.md", "# Converted Word") - - fake_result = create_parse_result( - root=ResourceNode(type=NodeType.ROOT), - source_path=str(tmp_path / "report.docx"), - source_format="docx", - parser_name="WordParser", - parse_time=0.1, - ) - fake_result.temp_dir_path = mock_temp - - with patch( - "openviking.parse.parsers.directory.DirectoryParser._assign_parser", - ) as mock_assign: - from openviking.parse.parsers.word import WordParser as _Word - - mock_word = AsyncMock(spec=_Word) - mock_word.parse = AsyncMock(return_value=fake_result) - - def assign_side_effect(cf, registry): - if cf.path.suffix == ".docx": - return mock_word - return registry.get_parser_for_file(cf.path) - - mock_assign.side_effect = assign_side_effect - await parser.parse(str(tmp_path)) - - dir_name = tmp_path.name - found_md = any( - uri.endswith("report.md") and f"/{dir_name}/" in uri for uri in fake_fs.files - ) - assert found_md, f"report.md not found. Files: {list(fake_fs.files.keys())}" - - @pytest.mark.asyncio - async def test_xlsx_file_goes_through_parser(self, tmp_path: Path, parser, fake_fs) -> None: - """Excel (.xlsx) files should be processed by ExcelParser.parse().""" - (tmp_path / "data.xlsx").write_bytes(b"PK\x03\x04") - - mock_temp = fake_fs.create_temp_uri() - doc_dir = f"{mock_temp}/data" - await fake_fs.mkdir(mock_temp) - await fake_fs.mkdir(doc_dir) - await fake_fs.write_file(f"{doc_dir}/data.md", "# Converted Excel") - - fake_result = create_parse_result( - root=ResourceNode(type=NodeType.ROOT), - source_path=str(tmp_path / "data.xlsx"), - source_format="xlsx", - parser_name="ExcelParser", - parse_time=0.1, - ) - fake_result.temp_dir_path = mock_temp - - with patch( - "openviking.parse.parsers.directory.DirectoryParser._assign_parser", - ) as mock_assign: - from openviking.parse.parsers.excel import ExcelParser as _Excel - - mock_excel = AsyncMock(spec=_Excel) - mock_excel.parse = AsyncMock(return_value=fake_result) - - def assign_side_effect(cf, registry): - if cf.path.suffix in {".xlsx", ".xls", ".xlsm"}: - return mock_excel - return registry.get_parser_for_file(cf.path) - - mock_assign.side_effect = assign_side_effect - await parser.parse(str(tmp_path)) - - dir_name = tmp_path.name - found_md = any(uri.endswith("data.md") and f"/{dir_name}/" in uri for uri in fake_fs.files) - assert found_md, f"data.md not found. Files: {list(fake_fs.files.keys())}" - - @pytest.mark.asyncio - async def test_epub_file_goes_through_parser(self, tmp_path: Path, parser, fake_fs) -> None: - """EPub (.epub) files should be processed by EPubParser.parse().""" - (tmp_path / "book.epub").write_bytes(b"PK\x03\x04") - - mock_temp = fake_fs.create_temp_uri() - doc_dir = f"{mock_temp}/book" - await fake_fs.mkdir(mock_temp) - await fake_fs.mkdir(doc_dir) - await fake_fs.write_file(f"{doc_dir}/book.md", "# Converted EPub") - - fake_result = create_parse_result( - root=ResourceNode(type=NodeType.ROOT), - source_path=str(tmp_path / "book.epub"), - source_format="epub", - parser_name="EPubParser", - parse_time=0.1, - ) - fake_result.temp_dir_path = mock_temp - - with patch( - "openviking.parse.parsers.directory.DirectoryParser._assign_parser", - ) as mock_assign: - from openviking.parse.parsers.epub import EPubParser as _EPub - - mock_epub = AsyncMock(spec=_EPub) - mock_epub.parse = AsyncMock(return_value=fake_result) - - def assign_side_effect(cf, registry): - if cf.path.suffix == ".epub": - return mock_epub - return registry.get_parser_for_file(cf.path) - - mock_assign.side_effect = assign_side_effect - await parser.parse(str(tmp_path)) - - dir_name = tmp_path.name - found_md = any(uri.endswith("book.md") and f"/{dir_name}/" in uri for uri in fake_fs.files) - assert found_md, f"book.md not found. Files: {list(fake_fs.files.keys())}" - - @pytest.mark.asyncio - async def test_pptx_file_goes_through_parser(self, tmp_path: Path, parser, fake_fs) -> None: - """PowerPoint (.pptx) files should be processed by PowerPointParser.parse().""" - (tmp_path / "slides.pptx").write_bytes(b"PK\x03\x04") + @pytest.mark.parametrize( + ("filename", "converted_content"), + [ + ("report.docx", "# Converted Word"), + ("data.xlsx", "# Converted Excel"), + ("book.epub", "# Converted EPUB"), + ("slides.pptx", "# Converted PowerPoint"), + ], + ) + async def test_anydoc_file_goes_through_parser( + self, + tmp_path: Path, + parser, + fake_fs, + filename: str, + converted_content: str, + ) -> None: + """Every AnyDoc-backed format uses the same directory import path.""" + source = tmp_path / filename + source.write_bytes(b"PK\x03\x04") + stem = source.stem mock_temp = fake_fs.create_temp_uri() - doc_dir = f"{mock_temp}/slides" + doc_dir = f"{mock_temp}/{stem}" await fake_fs.mkdir(mock_temp) await fake_fs.mkdir(doc_dir) - await fake_fs.write_file(f"{doc_dir}/slides.md", "# Converted PowerPoint") + await fake_fs.write_file(f"{doc_dir}/{stem}.md", converted_content) fake_result = create_parse_result( root=ResourceNode(type=NodeType.ROOT), - source_path=str(tmp_path / "slides.pptx"), - source_format="pptx", - parser_name="PowerPointParser", + source_path=str(source), + source_format=source.suffix.lstrip("."), + parser_name="AnyDocParser", parse_time=0.1, ) fake_result.temp_dir_path = mock_temp @@ -574,24 +470,24 @@ async def test_pptx_file_goes_through_parser(self, tmp_path: Path, parser, fake_ with patch( "openviking.parse.parsers.directory.DirectoryParser._assign_parser", ) as mock_assign: - from openviking.parse.parsers.powerpoint import PowerPointParser as _PPT + from openviking.parse.parsers.anydoc import AnyDocParser - mock_ppt = AsyncMock(spec=_PPT) - mock_ppt.parse = AsyncMock(return_value=fake_result) + mock_anydoc = AsyncMock(spec=AnyDocParser) + mock_anydoc.parse = AsyncMock(return_value=fake_result) - def assign_side_effect(cf, registry): - if cf.path.suffix == ".pptx": - return mock_ppt - return registry.get_parser_for_file(cf.path) + def assign_side_effect(candidate, registry): + if candidate.path == source: + return mock_anydoc + return registry.get_parser_for_file(candidate.path) mock_assign.side_effect = assign_side_effect await parser.parse(str(tmp_path)) - dir_name = tmp_path.name found_md = any( - uri.endswith("slides.md") and f"/{dir_name}/" in uri for uri in fake_fs.files + uri.endswith(f"/{stem}.md") and f"/{tmp_path.name}/" in uri + for uri in fake_fs.files ) - assert found_md, f"slides.md not found. Files: {list(fake_fs.files.keys())}" + assert found_md, f"{stem}.md not found. Files: {list(fake_fs.files)}" @pytest.mark.asyncio async def test_zip_file_goes_through_parser(self, tmp_path: Path, parser, fake_fs) -> None: diff --git a/tests/parse/test_directory_parser_routing.py b/tests/parse/test_directory_parser_routing.py index ca75798cf6..262c546187 100644 --- a/tests/parse/test_directory_parser_routing.py +++ b/tests/parse/test_directory_parser_routing.py @@ -27,14 +27,11 @@ DirectoryScanResult, scan_directory, ) -from openviking.parse.parsers.epub import EPubParser -from openviking.parse.parsers.excel import ExcelParser +from openviking.parse.parsers.anydoc import AnyDocParser from openviking.parse.parsers.html import HTMLParser from openviking.parse.parsers.markdown import MarkdownParser from openviking.parse.parsers.pdf import PDFParser -from openviking.parse.parsers.powerpoint import PowerPointParser from openviking.parse.parsers.text import TextParser -from openviking.parse.parsers.word import WordParser from openviking.parse.parsers.zip_parser import ZipParser from openviking.parse.registry import ParserRegistry @@ -72,13 +69,14 @@ def tmp_all_parsers(tmp_path: Path) -> Path: notes.txt -> TextParser log.text -> TextParser office/ - report.docx -> WordParser - data.xlsx -> ExcelParser - legacy.xls -> ExcelParser - macro.xlsm -> ExcelParser - slides.pptx -> PowerPointParser + legacy.doc -> AnyDocParser + report.docx -> AnyDocParser + data.xlsx -> AnyDocParser + legacy.xls -> AnyDocParser + macro.xlsm -> AnyDocParser + slides.pptx -> AnyDocParser books/ - book.epub -> EPubParser + book.epub -> AnyDocParser archives/ bundle.zip -> ZipParser code/ @@ -123,6 +121,7 @@ def tmp_all_parsers(tmp_path: Path) -> Path: (tmp_path / "config" / "rules.toml").write_text("[section]", encoding="utf-8") (tmp_path / "office").mkdir() + (tmp_path / "office" / "legacy.doc").write_bytes(b"\xd0\xcf\x11\xe0") (tmp_path / "office" / "report.docx").write_bytes(b"PK\x03\x04") (tmp_path / "office" / "data.xlsx").write_bytes(b"PK\x03\x04") (tmp_path / "office" / "legacy.xls").write_bytes(b"\xd0\xcf\x11\xe0") @@ -156,12 +155,13 @@ class TestParserSelection: ".pdf": PDFParser, ".txt": TextParser, ".text": TextParser, - ".docx": WordParser, - ".xlsx": ExcelParser, - ".xls": ExcelParser, - ".xlsm": ExcelParser, - ".epub": EPubParser, - ".pptx": PowerPointParser, + ".doc": AnyDocParser, + ".docx": AnyDocParser, + ".xlsx": AnyDocParser, + ".xls": AnyDocParser, + ".xlsm": AnyDocParser, + ".epub": AnyDocParser, + ".pptx": AnyDocParser, ".zip": ZipParser, } @@ -240,10 +240,18 @@ class TestParserCanParse: (HTMLParser, ["page.html", "site.htm"]), (PDFParser, ["paper.pdf"]), (TextParser, ["notes.txt", "log.text"]), - (WordParser, ["report.docx"]), - (ExcelParser, ["data.xlsx", "legacy.xls", "book.xlsm"]), - (EPubParser, ["book.epub"]), - (PowerPointParser, ["slides.pptx"]), + ( + AnyDocParser, + [ + "legacy.doc", + "report.docx", + "data.xlsx", + "legacy.xls", + "book.xlsm", + "book.epub", + "slides.pptx", + ], + ), (ZipParser, ["archive.zip"]), ], ) @@ -261,10 +269,7 @@ def test_can_parse_returns_true(self, parser_cls: type, filenames: List[str]) -> (HTMLParser, ["file.md", "file.pdf", "file.txt"]), (PDFParser, ["file.md", "file.txt", "file.html"]), (TextParser, ["file.md", "file.html", "file.pdf"]), - (WordParser, ["file.pdf", "file.xlsx", "file.txt"]), - (ExcelParser, ["file.docx", "file.pdf", "file.txt"]), - (EPubParser, ["file.pdf", "file.docx", "file.zip"]), - (PowerPointParser, ["file.pdf", "file.docx", "file.txt"]), + (AnyDocParser, ["file.pdf", "file.md", "file.zip"]), (ZipParser, ["file.rar", "file.pdf", "file.docx"]), ], ) diff --git a/tests/parse/test_document_parser_threading.py b/tests/parse/test_document_parser_threading.py index 35e9165a05..825fbf2c6e 100644 --- a/tests/parse/test_document_parser_threading.py +++ b/tests/parse/test_document_parser_threading.py @@ -1,9 +1,7 @@ # Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. # SPDX-License-Identifier: AGPL-3.0 -"""Regression tests for offloading synchronous document conversions.""" +"""Contract tests for the unified AnyDoc document parser.""" -import sys -import zipfile from pathlib import Path from types import SimpleNamespace from typing import Any, Callable @@ -11,10 +9,43 @@ import pytest from openviking.parse.base import NodeType, ResourceNode, create_parse_result -from openviking.parse.parsers import epub, excel, legacy_doc, powerpoint, word +from openviking.parse.parsers import anydoc -def _stub_markdown_parse(parser) -> dict[str, Any]: +class _FakeStorage: + def __init__(self, media_dir: Path) -> None: + self.media_dir = media_dir + self.saved: list[tuple[str, bytes, str, str]] = [] + + def save_image( + self, + resource_name: str, + data: bytes, + *, + filename: str, + extension: str, + ) -> Path: + self.saved.append((resource_name, data, filename, extension)) + return self.media_dir / resource_name / "images" / f"{filename}{extension}" + + +def _model(**values: Any) -> SimpleNamespace: + return SimpleNamespace(**values) + + +def _text(value: str) -> SimpleNamespace: + return _model(kind="text", text=value, style=None) + + +def _paragraph(*content: SimpleNamespace) -> SimpleNamespace: + return _model(kind="paragraph", content=list(content)) + + +def _document(*blocks: SimpleNamespace, assets=None, notes=None) -> SimpleNamespace: + return _model(blocks=list(blocks), assets=assets or [], notes=notes or []) + + +def _stub_markdown_parse(parser: anydoc.AnyDocParser) -> dict[str, Any]: seen: dict[str, Any] = {} async def parse_content( @@ -23,10 +54,12 @@ async def parse_content( instruction: str = "", **kwargs, ): - seen["content"] = content - seen["source_path"] = source_path - seen["instruction"] = instruction - seen["kwargs"] = kwargs + seen.update( + content=content, + source_path=source_path, + instruction=instruction, + kwargs=kwargs, + ) return create_parse_result( root=ResourceNode(type=NodeType.ROOT), source_path=source_path, @@ -34,244 +67,164 @@ async def parse_content( parser_name="MarkdownParser", ) - parser._md_parser.parse_content = parse_content + parser._markdown_parser.parse_content = parse_content return seen -def _patch_to_thread(monkeypatch, module) -> list[tuple[Callable[..., Any], tuple, dict]]: - calls: list[tuple[Callable[..., Any], tuple, dict]] = [] +@pytest.mark.asyncio +async def test_anydoc_parser_offloads_conversion_and_forwards_markdown_options( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + parser = anydoc.AnyDocParser() + seen = _stub_markdown_parse(parser) + calls: list[tuple[Callable[..., Any], tuple[Any, ...], dict[str, Any]]] = [] async def fake_to_thread(func, /, *args, **kwargs): calls.append((func, args, kwargs)) return func(*args, **kwargs) - monkeypatch.setattr(module.asyncio, "to_thread", fake_to_thread) - return calls - - -@pytest.mark.asyncio -async def test_word_parser_offloads_docx_conversion(monkeypatch, tmp_path: Path): - parser = word.WordParser() - seen = _stub_markdown_parse(parser) - calls = _patch_to_thread(monkeypatch, word) - fake_docx = SimpleNamespace() - monkeypatch.setitem(sys.modules, "docx", fake_docx) - - def convert(path: Path, docx_module, resource_name=None, storage=None) -> str: - assert docx_module is fake_docx - return "# converted docx" - - monkeypatch.setattr(parser, "_convert_to_markdown", convert) - source = tmp_path / "sample.docx" - source.write_bytes(b"placeholder") - - result = await parser.parse(source) - - # #2429 added resource_name/storage to the offloaded conversion call; assert - # the conversion was offloaded with the doc + docx module, without pinning the - # identity of the storage singleton. - assert len(calls) == 1 - func, args, _ = calls[0] - assert func is convert - assert args[0] == source - assert args[1] is fake_docx - assert seen["content"] == "# converted docx" - assert result.source_format == "docx" - assert result.parser_name == "WordParser" - - -@pytest.mark.asyncio -async def test_word_parser_forwards_original_name_to_markdown(monkeypatch, tmp_path: Path): - # On single-file upload the on-disk path is a temp name (upload_.docx) - # and the user's original filename arrives via source_name. WordParser must - # forward that name to MarkdownParser explicitly; otherwise parse_content can - # only fall back to the temp path and the resource ends up named upload_. - parser = word.WordParser() - seen = _stub_markdown_parse(parser) - _patch_to_thread(monkeypatch, word) - monkeypatch.setitem(sys.modules, "docx", SimpleNamespace()) - monkeypatch.setattr(parser, "_convert_to_markdown", lambda *args, **kwargs: "# converted docx") - - upload = tmp_path / "upload_abc123.docx" - upload.write_bytes(b"placeholder") - - await parser.parse(upload, source_name="季度报告.docx") - - forwarded = seen["kwargs"].get("source_name") or seen["kwargs"].get("resource_name") - assert forwarded == "季度报告.docx", seen["kwargs"] - - -@pytest.mark.asyncio -async def test_word_parser_forwards_no_split_to_markdown(monkeypatch, tmp_path: Path): - parser = word.WordParser() - seen = _stub_markdown_parse(parser) - _patch_to_thread(monkeypatch, word) - monkeypatch.setitem(sys.modules, "docx", SimpleNamespace()) - monkeypatch.setattr(parser, "_convert_to_markdown", lambda *args, **kwargs: "# converted docx") - - upload = tmp_path / "screenplay.docx" - upload.write_bytes(b"placeholder") - - await parser.parse(upload, split_content=False) - - assert seen["kwargs"]["split_content"] is False - - -@pytest.mark.asyncio -async def test_excel_parser_offloads_xlsx_conversion(monkeypatch, tmp_path: Path): - parser = excel.ExcelParser() - seen = _stub_markdown_parse(parser) - calls = _patch_to_thread(monkeypatch, excel) - fake_openpyxl = SimpleNamespace() - monkeypatch.setitem(sys.modules, "openpyxl", fake_openpyxl) - - def convert(path: Path, openpyxl_module) -> str: - assert openpyxl_module is fake_openpyxl - return "# converted xlsx" - - monkeypatch.setattr(parser, "_convert_to_markdown", convert) - source = tmp_path / "sample.xlsx" - source.write_bytes(b"placeholder") - - result = await parser.parse(source) - - assert calls == [(convert, (source, fake_openpyxl), {})] - assert seen["content"] == "# converted xlsx" - assert result.source_format == "xlsx" - assert result.parser_name == "ExcelParser" - - -@pytest.mark.asyncio -async def test_excel_parser_offloads_xls_conversion(monkeypatch, tmp_path: Path): - parser = excel.ExcelParser() - seen = _stub_markdown_parse(parser) - calls = _patch_to_thread(monkeypatch, excel) - - def convert(path: Path) -> str: - return "# converted xls" - - monkeypatch.setattr(parser, "_convert_xls_to_markdown", convert) - source = tmp_path / "sample.xls" - source.write_bytes(b"placeholder") - - result = await parser.parse(source) - - assert calls == [(convert, (source,), {})] - assert seen["content"] == "# converted xls" - assert result.source_format == "xls" - assert result.parser_name == "ExcelParser" - - -@pytest.mark.asyncio -async def test_powerpoint_parser_offloads_pptx_conversion(monkeypatch, tmp_path: Path): - parser = powerpoint.PowerPointParser() - seen = _stub_markdown_parse(parser) - calls = _patch_to_thread(monkeypatch, powerpoint) - fake_pptx = SimpleNamespace() - monkeypatch.setitem(sys.modules, "pptx", fake_pptx) - - def convert(path: Path, pptx_module) -> str: - assert pptx_module is fake_pptx - return "# converted pptx" - - monkeypatch.setattr(parser, "_convert_to_markdown", convert) - source = tmp_path / "sample.pptx" - source.write_bytes(b"placeholder") - - result = await parser.parse(source) - - assert calls == [(convert, (source, fake_pptx), {})] - assert seen["content"] == "# converted pptx" - assert result.source_format == "pptx" - assert result.parser_name == "PowerPointParser" - - -@pytest.mark.asyncio -async def test_epub_parser_offloads_epub_conversion(monkeypatch, tmp_path: Path): - parser = epub.EPubParser() - seen = _stub_markdown_parse(parser) - calls = _patch_to_thread(monkeypatch, epub) - - def convert(path: Path) -> str: - return "# converted epub" - - monkeypatch.setattr(parser, "_convert_to_markdown", convert) - source = tmp_path / "sample.epub" - source.write_bytes(b"placeholder") - - result = await parser.parse(source) - - assert calls == [(convert, (source,), {})] - assert seen["content"] == "# converted epub" - assert result.source_format == "epub" - assert result.parser_name == "EPubParser" - - -@pytest.mark.asyncio -async def test_legacy_doc_parser_offloads_doc_extraction(monkeypatch, tmp_path: Path): - parser = legacy_doc.LegacyDocParser() - seen = _stub_markdown_parse(parser) - calls = _patch_to_thread(monkeypatch, legacy_doc) - - def extract(path: Path) -> str: - return "# converted doc" + converted = anydoc._RenderedDocument( + markdown="# Converted\n", + detected_format="docx", + warnings=["conversion warning"], + assets_referenced=2, + images_extracted=1, + ) + monkeypatch.setattr(anydoc.asyncio, "to_thread", fake_to_thread) + monkeypatch.setattr(parser, "_convert", lambda *args, **kwargs: converted) + monkeypatch.setattr(anydoc, "version", lambda _package: "0.1.8") + storage = _FakeStorage(tmp_path / "media") + monkeypatch.setattr("openviking_cli.utils.storage.get_storage", lambda: storage) - monkeypatch.setattr(parser, "_extract_text", extract) - source = tmp_path / "sample.doc" + source = tmp_path / "upload.docx" source.write_bytes(b"placeholder") - - result = await parser.parse(source) - - assert calls == [(extract, (source,), {})] - assert seen["content"] == "# converted doc" - assert result.source_format == "doc" - assert result.parser_name == "LegacyDocParser" - - -@pytest.mark.asyncio -async def test_legacy_doc_parser_routes_ooxml_payload_to_word_parser(monkeypatch, tmp_path: Path): - parser = legacy_doc.LegacyDocParser() - _stub_markdown_parse(parser) - source = tmp_path / "mislabeled.doc" - with zipfile.ZipFile(source, "w") as archive: - archive.writestr("[Content_Types].xml", "") - archive.writestr("word/document.xml", "") - - seen: dict[str, Any] = {} - - async def parse_word(self, source_path, instruction="", **kwargs): - seen["source"] = source_path - seen["instruction"] = instruction - seen["kwargs"] = kwargs - return create_parse_result( - root=ResourceNode(type=NodeType.ROOT), - source_path=str(source_path), - source_format="docx", - parser_name="WordParser", - ) - - monkeypatch.setattr(word.WordParser, "parse", parse_word) - + caller_media = tmp_path / "caller-media" result = await parser.parse( source, instruction="preserve tables", - source_name="mislabeled.doc", + source_name="季度报告.docx", + allowed_media_dirs=[caller_media], + split_content=False, ) + assert len(calls) == 1 + assert calls[0][0] is parser._convert + assert calls[0][1] == (source,) + assert calls[0][2] == { + "resource_name": "季度报告.docx", + "storage": storage, + } assert seen == { - "source": source, + "content": "# Converted\n", + "source_path": str(source), "instruction": "preserve tables", - "kwargs": {"source_name": "mislabeled.doc"}, + "kwargs": { + "base_dir": tmp_path, + "allowed_media_dirs": [caller_media, storage.media_dir], + "source_name": "季度报告.docx", + "split_content": False, + }, } assert result.source_format == "docx" - assert result.parser_name == "WordParser" + assert result.parser_name == "AnyDocParser" + assert result.warnings == ["conversion warning"] + assert result.meta["images_extracted"] == 1 + + +def test_anydoc_renderer_preserves_image_position_and_saves_repeated_asset_once( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + storage = _FakeStorage(tmp_path / "media") + asset = _model( + id=7, + media_type="image/png", + origin_part="word/media/image.png", + data=b"valid-image", + ) + embedded = _model( + kind="image", + alt="diagram", + source=_model(kind="asset", asset_id=7), + ) + external = _model( + kind="image", + alt="remote", + source=_model(kind="external", url="https://example.com/image.png"), + ) + document = _document( + _paragraph(_text("before "), embedded, _text(" middle "), embedded, external), + assets=[asset], + ) + monkeypatch.setattr(anydoc, "is_valid_image", lambda *_args: True) + renderer = anydoc._AnyDocMarkdownRenderer( + document, + source_format="docx", + resource_name="report", + storage=storage, + ) + markdown = renderer.render() -@pytest.mark.asyncio -async def test_legacy_doc_parser_rejects_non_word_zip_payload(tmp_path: Path): - source = tmp_path / "not-a-word-document.doc" - with zipfile.ZipFile(source, "w") as archive: - archive.writestr("payload.bin", b"binary") + image_ref = "report/images/anydoc_asset_7.png" + assert markdown == ( + f"before ![diagram]({image_ref}) middle ![diagram]({image_ref})" + "![remote](https://example.com/image.png)\n" + ) + assert storage.saved == [("report", b"valid-image", "anydoc_asset_7", ".png")] + assert renderer.images_extracted == 1 + assert renderer.assets_referenced == {7} + + +def test_anydoc_renderer_keeps_ppt_speaker_notes_in_document_order(tmp_path: Path) -> None: + heading = _model(kind="heading", level=1, content=[_text("Slide One")], anchor=None) + notes = _model(kind="block_quote", blocks=[_paragraph(_text("Presenter detail"))]) + document = _document(heading, _paragraph(_text("Slide body")), notes) + + markdown = anydoc._AnyDocMarkdownRenderer( + document, + source_format="pptx", + resource_name="slides", + storage=_FakeStorage(tmp_path / "media"), + ).render() + + assert markdown == ("# Slide One\n\nSlide body\n\n### Speaker Notes\n\nPresenter detail\n") + + +def test_anydoc_renderer_warns_and_keeps_alt_for_unusable_assets(tmp_path: Path) -> None: + document = _document( + _paragraph( + _model( + kind="image", + alt="attachment", + source=_model(kind="asset", asset_id=3), + ), + _text(" "), + _model( + kind="image", + alt="missing preview", + source=_model(kind="unavailable"), + ), + ), + assets=[ + _model( + id=3, + media_type="application/octet-stream", + origin_part="embedding.bin", + data=b"not-an-image", + ) + ], + ) + renderer = anydoc._AnyDocMarkdownRenderer( + document, + source_format="xlsx", + resource_name="book", + storage=_FakeStorage(tmp_path / "media"), + ) - with pytest.raises(ValueError, match="ZIP package"): - await legacy_doc.LegacyDocParser().parse(source) + assert renderer.render() == "attachment missing preview\n" + assert renderer.warnings == [ + "AnyDoc asset 3 is not an image (application/octet-stream); kept as alt text", + "Embedded image is unavailable: missing preview", + ] diff --git a/tests/parse/test_excel_process_pool.py b/tests/parse/test_excel_process_pool.py deleted file mode 100644 index 2f376a7d54..0000000000 --- a/tests/parse/test_excel_process_pool.py +++ /dev/null @@ -1,290 +0,0 @@ -# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. -# SPDX-License-Identifier: AGPL-3.0 -"""Tests for ExcelParser._should_use_process_pool, the config-gated routing decision -that decides whether Excel→Markdown conversion + layout planning run in a -ProcessPoolExecutor child process instead of the main process's event loop. - -A real ProcessPoolExecutor is not exercised here: spawning a child process per -test is slow and orthogonal to this decision, and the worker itself -(_build_excel_layout_in_process) is a thin, side-effect-free function that only -touches its own arguments. This covers the routing decision that gates it. -""" - -from pathlib import Path - -import pytest - -from openviking.parse.parsers import excel as excel_module -from openviking.parse.parsers.excel import ( - ExcelParser, - _EXCEL_PROCESS_POOL_MIN_BYTES, -) -from openviking_cli.utils.config.parser_config import ( - ExcelConfig, - MarkdownConfig, - ParserConfig, -) - - -class TestShouldUseProcessPool: - def _parser(self, **excel_kwargs) -> ExcelParser: - return ExcelParser(config=ExcelConfig(**excel_kwargs)) - - def _make_file(self, tmp_path: Path, suffix: str = ".xlsx", size: int = 10) -> Path: - path = tmp_path / f"sheet{suffix}" - path.write_bytes(b"x" * size) - return path - - def test_disabled_by_default(self, tmp_path: Path): - path = self._make_file(tmp_path, size=_EXCEL_PROCESS_POOL_MIN_BYTES) - assert self._parser()._should_use_process_pool(path, {}) is False - - def test_xls_never_uses_process_pool(self, tmp_path: Path): - # Legacy .xls goes through xlrd, not the openpyxl path the worker assumes. - path = self._make_file(tmp_path, suffix=".xls", size=_EXCEL_PROCESS_POOL_MIN_BYTES) - assert ( - self._parser(enable_process_pool=True)._should_use_process_pool(path, {}) - is False - ) - - @pytest.mark.parametrize( - "kwargs", - [ - {"enable_link_rewrite": True}, - {"base_dir": Path(".")}, - {"allowed_media_dirs": [Path(".")]}, - ], - ) - def test_link_or_media_rewrite_kwargs_disable_process_pool(self, tmp_path, kwargs): - # The worker never touches VikingFS/base_dir-relative media, so any parse - # that needs link/media rewriting must stay in-process. - path = self._make_file(tmp_path, size=_EXCEL_PROCESS_POOL_MIN_BYTES) - assert ( - self._parser(enable_process_pool=True)._should_use_process_pool(path, kwargs) - is False - ) - - def test_below_min_bytes_returns_false(self, tmp_path): - path = self._make_file(tmp_path, size=_EXCEL_PROCESS_POOL_MIN_BYTES - 1) - assert ( - self._parser(enable_process_pool=True)._should_use_process_pool(path, {}) - is False - ) - - def test_at_or_above_min_bytes_returns_true(self, tmp_path): - path = self._make_file(tmp_path, size=_EXCEL_PROCESS_POOL_MIN_BYTES) - assert ( - self._parser(enable_process_pool=True)._should_use_process_pool(path, {}) - is True - ) - - -@pytest.mark.asyncio -async def test_process_pool_forwards_no_split_layout_flag( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -): - path = tmp_path / "sheet.xlsx" - path.write_bytes(b"xlsx") - parser = ExcelParser(config=ExcelConfig(enable_process_pool=True)) - monkeypatch.setattr( - parser._md_parser, - "_create_temp_uri", - lambda: "viking://temp/test/excel", - ) - captured = {} - - class _CapturingLoop: - def run_in_executor(self, _executor, callback): - captured["callback"] = callback - raise RuntimeError("layout callback captured") - - monkeypatch.setattr(excel_module.asyncio, "get_running_loop", lambda: _CapturingLoop()) - monkeypatch.setattr(excel_module, "_get_excel_layout_executor", lambda _workers: object()) - - with pytest.raises(RuntimeError, match="layout callback captured"): - await parser._parse_existing_path_process_pool(path, split_content=False) - - callback = captured["callback"] - assert callback.keywords["layout_kwargs"]["split_content"] is False - - -class TestExcelConfig: - def test_defaults(self): - cfg = ExcelConfig() - assert cfg.enable_process_pool is False - assert cfg.process_pool_workers == 2 - - def test_validate_rejects_zero_workers(self): - cfg = ExcelConfig(process_pool_workers=0) - with pytest.raises(ValueError, match="process_pool_workers"): - cfg.validate() - - def test_openviking_config_accepts_excel_section(self): - from openviking_cli.utils.config.open_viking_config import OpenVikingConfig - - cfg = OpenVikingConfig.from_dict( - { - "excel": { - "enable_process_pool": True, - "process_pool_workers": 8, - } - } - ) - assert cfg.excel.enable_process_pool is True - assert cfg.excel.process_pool_workers == 8 - - -class TestExcelSectioningInheritance: - """Excel used to be registered with ``config.markdown``. - - A dedicated ``parsers.excel`` section must not silently change sectioning, - because section boundaries decide node structure and stable Viking URIs. - """ - - def test_unset_sectioning_follows_markdown(self): - markdown = MarkdownConfig(max_section_size=512, max_section_chars=2000) - resolved = ExcelConfig.from_dict({}).with_sectioning_defaults_from(markdown) - assert resolved.max_section_size == 512 - assert resolved.max_section_chars == 2000 - - def test_explicit_excel_values_win(self): - markdown = MarkdownConfig(max_section_size=512, max_section_chars=2000) - resolved = ExcelConfig.from_dict( - {"max_section_size": 1024} - ).with_sectioning_defaults_from(markdown) - assert resolved.max_section_size == 1024 - # Absent from parsers.excel, so this one keeps following Markdown. - assert resolved.max_section_chars == 2000 - - def test_explicit_value_equal_to_default_is_not_overwritten(self): - """"Excel keeps 2048" is a legitimate config even though 2048 is the default. - - Comparing against class defaults cannot distinguish it from an absent - key, which would make the documented "explicit values win" contract - false for exactly this case. - """ - markdown = MarkdownConfig(max_section_size=512) - resolved = ExcelConfig.from_dict( - {"max_section_size": 2048} - ).with_sectioning_defaults_from(markdown) - assert resolved.max_section_size == 2048 - - def test_process_pool_knobs_are_never_inherited(self): - markdown = MarkdownConfig(max_section_size=512) - resolved = ExcelConfig.from_dict( - {"enable_process_pool": True, "process_pool_workers": 8} - ).with_sectioning_defaults_from(markdown) - assert resolved.enable_process_pool is True - assert resolved.process_pool_workers == 8 - assert resolved.max_section_size == 512 - - def test_default_markdown_leaves_excel_unchanged(self): - resolved = ExcelConfig.from_dict({}).with_sectioning_defaults_from(MarkdownConfig()) - assert resolved == ExcelConfig() - - def test_missing_markdown_config_is_tolerated(self): - assert ExcelConfig.from_dict({}).with_sectioning_defaults_from(None) == ExcelConfig() - - def test_hand_built_config_is_treated_as_fully_explicit(self): - """A config built without from_dict carries no key provenance.""" - markdown = MarkdownConfig(max_section_size=512) - resolved = ExcelConfig(max_section_size=2048).with_sectioning_defaults_from(markdown) - assert resolved.max_section_size == 2048 - - def test_provenance_does_not_affect_equality(self): - assert ExcelConfig.from_dict({}) == ExcelConfig() - assert ExcelConfig.from_dict({"max_section_size": 2048}) == ExcelConfig() - - def test_absent_excel_section_still_inherits_through_full_config(self): - """Deployments predating parsers.excel must keep their Excel sectioning.""" - from openviking_cli.utils.config.open_viking_config import OpenVikingConfig - - cfg = OpenVikingConfig.from_dict({"markdown": {"max_section_size": 512}}) - resolved = cfg.excel.with_sectioning_defaults_from(cfg.markdown) - assert resolved.max_section_size == 512 - - def test_key_provenance_does_not_leak_into_serialization(self): - """Provenance is bookkeeping, not config: it must not reach any output. - - A dataclass field would appear in asdict/model_dump and, being a - frozenset, would break JSON serialization for callers. - """ - import dataclasses - import json - - cfg = ExcelConfig.from_dict({"max_section_size": 2048}) - dumped = dataclasses.asdict(cfg) - assert "_explicit_fields" not in dumped - assert not any(key.startswith("_") for key in dumped), sorted(dumped) - json.dumps(dumped) - - def test_config_data_cannot_forge_key_provenance(self): - """Provenance must not be settable from a config file.""" - with pytest.raises(ValueError): - ExcelConfig.from_dict({"_explicit_fields": ["max_section_size"]}) - - def test_replace_and_copy_preserve_key_provenance(self): - import copy - import dataclasses - - markdown = MarkdownConfig(max_section_size=512) - cfg = ExcelConfig.from_dict({"max_section_size": 2048}) - - for variant in ( - dataclasses.replace(cfg, enable_process_pool=True), - copy.copy(cfg), - copy.deepcopy(cfg), - ): - resolved = variant.with_sectioning_defaults_from(markdown) - assert resolved.max_section_size == 2048 - - def test_absent_excel_section_inherits_through_parser_loader(self): - """load_parser_configs_from_dict is a second config entry point.""" - from openviking_cli.utils.config.parser_config import ( - load_parser_configs_from_dict, - ) - - configs = load_parser_configs_from_dict({"markdown": {"max_section_size": 512}}) - resolved = configs["excel"].with_sectioning_defaults_from(configs["markdown"]) - assert resolved.max_section_size == 512 - - def test_explicit_excel_section_wins_through_full_config(self): - from openviking_cli.utils.config.open_viking_config import OpenVikingConfig - - cfg = OpenVikingConfig.from_dict( - { - "markdown": {"max_section_size": 512}, - "excel": {"max_section_size": 2048, "enable_process_pool": True}, - } - ) - resolved = cfg.excel.with_sectioning_defaults_from(cfg.markdown) - assert resolved.max_section_size == 2048 - assert resolved.enable_process_pool is True - - def test_registry_resolves_excel_against_markdown(self, monkeypatch): - from types import SimpleNamespace - - from openviking.parse import registry as registry_module - - config = SimpleNamespace( - text=ParserConfig(), - markdown=MarkdownConfig(max_section_size=512, max_section_chars=2000), - pdf=ParserConfig(), - html=ParserConfig(), - excel=ExcelConfig.from_dict({"enable_process_pool": True}), - image=ParserConfig(), - ) - monkeypatch.setattr( - "openviking_cli.utils.config.get_openviking_config", - lambda: config, - ) - monkeypatch.setattr(registry_module, "_default_registry", None) - - excel_parser = registry_module.get_registry().get_parser_for_file("book.xlsx") - - assert excel_parser.config.enable_process_pool is True - assert excel_parser.config.max_section_size == 512 - # The inner MarkdownParser is what actually sections the converted sheet. - assert excel_parser._md_parser.config.max_section_size == 512 - assert excel_parser._md_parser.config.max_section_chars == 2000 diff --git a/tests/parse/test_markdown_no_split.py b/tests/parse/test_markdown_no_split.py index 62f5d18a6f..287dd295a3 100644 --- a/tests/parse/test_markdown_no_split.py +++ b/tests/parse/test_markdown_no_split.py @@ -103,14 +103,14 @@ async def test_pdf_no_split_converts_to_one_complete_markdown( content = _long_markdown(with_headings=True) fake_fs = _FakeVikingFS() parser = PDFParser( - PDFConfig(strategy="local", max_section_size=32, max_section_chars=128) + PDFConfig(max_section_size=32, max_section_chars=128) ) markdown_parser = parser._get_markdown_parser() monkeypatch.setattr(markdown_parser, "_get_viking_fs", lambda: fake_fs) monkeypatch.setattr( parser, "_convert_to_markdown", - AsyncMock(return_value=(content, {})), + AsyncMock(return_value=(content, {}, [])), ) monkeypatch.setattr( "openviking_cli.utils.storage.get_storage", diff --git a/tests/parse/test_parser_config_wiring.py b/tests/parse/test_parser_config_wiring.py index aaae5cc4c5..fe199a89a2 100644 --- a/tests/parse/test_parser_config_wiring.py +++ b/tests/parse/test_parser_config_wiring.py @@ -9,7 +9,7 @@ def test_pdf_parser_passes_its_config_to_nested_markdown_parser(): - parser = PDFParser(PDFConfig(strategy="local", max_section_size=2222, max_section_chars=5555)) + parser = PDFParser(PDFConfig(max_section_size=2222, max_section_chars=5555)) markdown_parser = parser._get_markdown_parser() diff --git a/tests/parse/test_pdf_bookmark_extraction.py b/tests/parse/test_pdf_bookmark_extraction.py index 441540e516..da05f50961 100644 --- a/tests/parse/test_pdf_bookmark_extraction.py +++ b/tests/parse/test_pdf_bookmark_extraction.py @@ -1,535 +1,193 @@ # Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. # SPDX-License-Identifier: AGPL-3.0 -""" -Tests for PDF bookmark/outline extraction in PDFParser. +"""Contract tests for pdf-inspector text and pdfplumber image extraction.""" -Verifies that _extract_bookmarks correctly extracts bookmark entries -and that _convert_local injects them as markdown headings. -""" - -from contextlib import nullcontext +import sys from pathlib import Path from types import SimpleNamespace -from unittest.mock import MagicMock, patch +from typing import Any import pytest -from openviking.parse.parsers.pdf import PDFParser - - -def _make_page(*, pageid=None, objid=None): - """Create a minimal page stub for bookmark extraction tests.""" - return SimpleNamespace(page_obj=SimpleNamespace(pageid=pageid, objid=objid)) - - -def _make_ref(objid): - """Create a minimal PDF object reference stub.""" - return SimpleNamespace(objid=objid) - - -class _FakePDFStream(dict): - """Minimal pdfminer PDFStream-like object.""" - - def __init__(self, data: bytes, subtype: str = "Image"): - super().__init__({"Subtype": SimpleNamespace(name=subtype)}) - self._data = data - - def get_data(self): - return self._data - - -class _FakePDFObjectRef: - """Minimal PDF object reference with a resolve() method.""" - - def __init__(self, stream): - self._stream = stream - - def resolve(self): - return self._stream - - -def _make_image_page(xobjects): - return SimpleNamespace(page_obj=SimpleNamespace(resources={"XObject": xobjects})) - - -class _FakeRenderedImage: - """Minimal rendered image stub that writes deterministic PNG bytes.""" - - def __init__(self, data: bytes): - self._data = data - - def save(self, buffer, format: str): - assert format == "PNG" - buffer.write(self._data) - - -class _FakeCroppedPage: - """Minimal cropped page stub for image extraction tests.""" - - def __init__(self, rendered_image: _FakeRenderedImage): - self._rendered_image = rendered_image - self.resolution_calls = [] - - def to_image(self, resolution: int): - self.resolution_calls.append(resolution) - return self._rendered_image - - -class _FakeImagePage: - """Minimal pdfplumber page stub for _extract_image_from_page tests.""" - - def __init__(self, *, width: int = 100, height: int = 200, image_bytes: bytes = b"png-bytes"): - self.width = width - self.height = height - self.crop_calls = [] - self.cropped_page = _FakeCroppedPage(_FakeRenderedImage(image_bytes)) - - def crop(self, bbox): - self.crop_calls.append(bbox) - return self.cropped_page - - -class _FakePage: - """Minimal pdfplumber page stub for _convert_local tests.""" - - def __init__(self, text: str): - self._text = text - self.images = [] - self.close_count = 0 - self.flush_count = 0 - - def extract_text(self): - return self._text - - def extract_tables(self): - return [] - - def flush_cache(self): - self.flush_count += 1 - - def close(self): - self.close_count += 1 - self.flush_cache() - - -class _FakeFontPage(_FakePage): - """Minimal page stub with character layout data for font heading detection.""" - - def __init__(self, *, page_number: int, chars: list[dict], height: int = 1000): - super().__init__("") - self.page_number = page_number - self.chars = chars - self.height = height - - -class TestExtractBookmarks: - """Test PDF bookmark extraction logic.""" - - def setup_method(self): - self.parser = PDFParser() - - def test_extract_bookmarks_with_outlines(self): - """Bookmarks are extracted from PDF outlines with correct levels and page mapping.""" - # Mock pdfplumber PDF object - mock_pdf = MagicMock() - - # Real pdfminer outlines point at page.page_obj.pageid - mock_pdf.pages = [_make_page(pageid=100), _make_page(pageid=200)] - - # Mock page reference objects for bookmark destinations - mock_ref1 = _make_ref(100) # Points to page 1 - mock_ref2 = _make_ref(200) # Points to page 2 - - # Mock document outlines: (level, title, dest, action, structelem) - mock_pdf.doc.get_outlines.return_value = [ - (1, "Chapter 1", [mock_ref1, "/Fit"], None, None), - (2, "Section 1.1", [mock_ref1, "/Fit"], None, None), - (1, "Chapter 2", [mock_ref2, "/Fit"], None, None), +from openviking.parse.parsers.pdf import ( + PDFParser, + _PdfImageExtraction, + _PdfTextExtraction, +) + + +class _FakeStorage: + def __init__(self, media_dir: Path) -> None: + self.media_dir = media_dir + self.saved: list[tuple[str, bytes, str, str]] = [] + + def save_image( + self, + resource_name: str, + data: bytes, + *, + filename: str, + extension: str, + ) -> Path: + self.saved.append((resource_name, data, filename, extension)) + return self.media_dir / resource_name / "images" / f"{filename}{extension}" + + +def test_pdf_inspector_owns_page_markdown_and_reports_ocr_pages( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + extraction = SimpleNamespace( + pages=[ + SimpleNamespace(page=0, markdown="# Introduction\n\nBody"), + SimpleNamespace(page=1, markdown=""), + ], + pages_needing_ocr=[2], + ocr_reasons_by_page=[SimpleNamespace(page=2, reasons=["no_text", "image_coverage"])], + pages_with_tables=[1], + pages_with_columns=[1], + is_complex=True, + ) + fake_module = SimpleNamespace( + extract_pages_markdown=lambda path: extraction, + ) + monkeypatch.setitem(sys.modules, "pdf_inspector", fake_module) + + result = PDFParser._extract_text_sync(tmp_path / "paper.pdf") + + assert result.pages == {1: "# Introduction\n\nBody", 2: ""} + assert result.warnings == ["PDF page 2 requires OCR: no_text, image_coverage"] + assert result.meta == { + "total_pages": 2, + "pages_processed": 1, + "pages_needing_ocr": [2], + "ocr_reasons_by_page": [{"page": 2, "reasons": ["no_text", "image_coverage"]}], + "pages_with_tables": [1], + "pages_with_columns": [1], + "is_complex_layout": True, + } + + +@pytest.mark.asyncio +async def test_pdf_markdown_keeps_page_order_and_places_images_after_text( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + parser = PDFParser() + text = _PdfTextExtraction( + pages={1: "# Page One", 2: ""}, + warnings=["PDF page 2 requires OCR: no_text"], + meta={"total_pages": 2, "pages_needing_ocr": [2]}, + ) + images = _PdfImageExtraction( + pages={ + 1: ["![first](doc/images/page1_img1.png)"], + 2: ["![scan](doc/images/page2_img1.png)"], + }, + warnings=[], + images_extracted=2, + images_deduplicated=0, + ) + monkeypatch.setattr(parser, "_extract_text_sync", lambda _path: text) + monkeypatch.setattr( + parser, + "_extract_images_sync", + lambda _path, _storage, _name: images, + ) + + markdown, meta, warnings = await parser._convert_to_markdown( + tmp_path / "paper.pdf", + storage=_FakeStorage(tmp_path / "media"), + resource_name="doc", + ) + + assert markdown == ( + "\n\n# Page One\n\n![first](doc/images/page1_img1.png)\n\n" + "\n\n![scan](doc/images/page2_img1.png)" + ) + assert warnings == ["PDF page 2 requires OCR: no_text"] + assert meta == { + "total_pages": 2, + "pages_needing_ocr": [2], + "library": "pdf-inspector", + "image_library": "pdfplumber", + "images_extracted": 2, + "images_deduplicated": 0, + } + + +@pytest.mark.asyncio +async def test_pdf_image_failure_does_not_replace_text_result( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + parser = PDFParser() + monkeypatch.setattr( + parser, + "_extract_text_sync", + lambda _path: _PdfTextExtraction( + pages={1: "Text survives"}, warnings=[], meta={"total_pages": 1} + ), + ) + + def fail_images(*_args: Any) -> _PdfImageExtraction: + raise RuntimeError("renderer unavailable") + + monkeypatch.setattr(parser, "_extract_images_sync", fail_images) + + markdown, meta, warnings = await parser._convert_to_markdown( + tmp_path / "paper.pdf", + storage=_FakeStorage(tmp_path / "media"), + resource_name="doc", + ) + + assert markdown == "\n\nText survives" + assert meta["images_extracted"] == 0 + assert warnings == ["PDF image extraction failed: renderer unavailable"] + + +def test_pdfplumber_images_are_deduplicated_per_page_and_cache_is_released( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + class FakePage: + width = 100 + height = 100 + images = [ + {"x0": 1, "top": 2, "x1": 30, "bottom": 40}, + {"x0": 1, "top": 2, "x1": 30, "bottom": 40}, ] - bookmarks = self.parser._extract_bookmarks(mock_pdf) - - assert len(bookmarks) == 3 - assert bookmarks[0] == {"title": "Chapter 1", "level": 1, "page_num": 1} - assert bookmarks[1] == {"title": "Section 1.1", "level": 2, "page_num": 1} - assert bookmarks[2] == {"title": "Chapter 2", "level": 1, "page_num": 2} - - def test_extract_bookmarks_falls_back_to_objid_mapping(self): - """Objid-based mapping remains supported for tests and alternate backends.""" - mock_pdf = MagicMock() - mock_pdf.pages = [_make_page(objid=100), _make_page(objid=200)] - - mock_pdf.doc.get_outlines.return_value = [ - (1, "Chapter 1", [_make_ref(100), "/Fit"], None, None), - (1, "Chapter 2", [_make_ref(200), "/Fit"], None, None), - ] - - bookmarks = self.parser._extract_bookmarks(mock_pdf) - assert [b["page_num"] for b in bookmarks] == [1, 2] - - def test_extract_bookmarks_no_outlines(self): - """Returns empty list when PDF has no outlines.""" - mock_pdf = MagicMock() - mock_pdf.pages = [] - mock_pdf.doc.get_outlines.return_value = [] - - bookmarks = self.parser._extract_bookmarks(mock_pdf) - assert bookmarks == [] - - def test_extract_bookmarks_no_get_outlines(self): - """Returns empty list when document has no get_outlines method.""" - mock_pdf = MagicMock() - mock_pdf.pages = [] - del mock_pdf.doc.get_outlines # Remove the method - - bookmarks = self.parser._extract_bookmarks(mock_pdf) - assert bookmarks == [] - - def test_extract_bookmarks_skips_empty_titles(self): - """Bookmarks with empty or whitespace-only titles are skipped.""" - mock_pdf = MagicMock() - mock_pdf.pages = [] - mock_pdf.doc.get_outlines.return_value = [ - (1, "", None, None, None), - (1, " ", None, None, None), - (1, "Valid Title", None, None, None), - ] - - bookmarks = self.parser._extract_bookmarks(mock_pdf) - assert len(bookmarks) == 1 - assert bookmarks[0]["title"] == "Valid Title" - - def test_extract_bookmarks_caps_level_at_6(self): - """Heading levels are capped at 6 for markdown compatibility.""" - mock_pdf = MagicMock() - mock_pdf.pages = [] - mock_pdf.doc.get_outlines.return_value = [ - (10, "Deep Heading", None, None, None), - ] - - bookmarks = self.parser._extract_bookmarks(mock_pdf) - assert bookmarks[0]["level"] == 6 - - def test_extract_bookmarks_unresolved_pages(self): - """Bookmarks with unresolvable destinations get page_num=None.""" - mock_pdf = MagicMock() - mock_pdf.pages = [] - mock_pdf.doc.get_outlines.return_value = [ - (1, "No Destination", None, None, None), - ] - - bookmarks = self.parser._extract_bookmarks(mock_pdf) - assert len(bookmarks) == 1 - assert bookmarks[0]["page_num"] is None - - def test_extract_bookmarks_integer_page_index(self): - """Bookmarks with integer destination (0-based) are resolved correctly.""" - mock_pdf = MagicMock() - mock_pdf.pages = [_make_page(pageid=100), _make_page(pageid=200)] - - # Integer page indices instead of object references - mock_pdf.doc.get_outlines.return_value = [ - (1, "Chapter 1", [0, "/Fit"], None, None), - (1, "Chapter 2", [1, "/Fit"], None, None), - ] - - bookmarks = self.parser._extract_bookmarks(mock_pdf) - assert len(bookmarks) == 2 - assert bookmarks[0]["page_num"] == 1 - assert bookmarks[0]["title"] == "Chapter 1" - assert bookmarks[1]["page_num"] == 2 - assert bookmarks[1]["title"] == "Chapter 2" - - def test_extract_bookmarks_integer_page_index_out_of_range(self): - """Out-of-range integer page indices are treated as unresolved.""" - mock_pdf = MagicMock() - mock_pdf.pages = [_make_page(pageid=100)] # Only 1 page - - mock_pdf.doc.get_outlines.return_value = [ - (1, "Valid", [0, "/Fit"], None, None), - (1, "Too High", [5, "/Fit"], None, None), - (1, "Negative", [-1, "/Fit"], None, None), - ] - - bookmarks = self.parser._extract_bookmarks(mock_pdf) - assert len(bookmarks) == 3 - assert bookmarks[0]["page_num"] == 1 - assert bookmarks[1]["page_num"] is None - assert bookmarks[2]["page_num"] is None - - def test_extract_bookmarks_exception_returns_empty(self): - """Returns empty list on unexpected exceptions (best-effort).""" - mock_pdf = MagicMock() - mock_pdf.pages = [] - mock_pdf.doc.get_outlines.side_effect = RuntimeError("Corrupt PDF") - - bookmarks = self.parser._extract_bookmarks(mock_pdf) - assert bookmarks == [] - - -class TestConvertLocalBookmarks: - """Test bookmark injection behavior in local PDF conversion.""" - - @pytest.mark.asyncio - async def test_convert_local_skips_unresolved_bookmarks(self): - parser = PDFParser() - fake_pdf = SimpleNamespace(pages=[_FakePage("Page one"), _FakePage("Page two")]) - fake_pdfplumber = SimpleNamespace(open=lambda _path: nullcontext(fake_pdf)) - - with ( - patch("openviking.parse.parsers.pdf.lazy_import", return_value=fake_pdfplumber), - patch.object( - parser, - "_extract_bookmarks", - return_value=[ - {"level": 1, "title": "Broken Bookmark", "page_num": None}, - {"level": 1, "title": "Chapter 2", "page_num": 2}, - ], - ), - ): - markdown, meta = await parser._convert_local( - "dummy.pdf", storage=MagicMock(), resource_name="dummy" - ) - - assert "Broken Bookmark" not in markdown - assert "\n# Chapter 2\n" in markdown - assert meta["bookmarks_found"] == 2 - assert meta["bookmarks_resolved"] == 1 - assert meta["bookmarks_unresolved"] == 1 - assert meta["headings_found"] == 1 - assert meta["heading_source"] == "bookmarks" - - @pytest.mark.asyncio - async def test_convert_local_falls_back_to_font_when_bookmarks_unresolved(self): - parser = PDFParser() - fake_pdf = SimpleNamespace(pages=[_FakePage("Page one"), _FakePage("Page two")]) - fake_pdfplumber = SimpleNamespace(open=lambda _path: nullcontext(fake_pdf)) - - with ( - patch("openviking.parse.parsers.pdf.lazy_import", return_value=fake_pdfplumber), - patch.object( - parser, - "_extract_bookmarks", - return_value=[{"level": 1, "title": "Broken Bookmark", "page_num": None}], - ), - patch.object( - parser, - "_detect_headings_by_font", - return_value=[{"level": 1, "title": "Font Heading", "page_num": 2}], - ), - ): - markdown, meta = await parser._convert_local( - "dummy.pdf", storage=MagicMock(), resource_name="dummy" - ) - - assert "Broken Bookmark" not in markdown - assert "\n# Font Heading\n" in markdown - assert meta["bookmarks_found"] == 1 - assert meta["bookmarks_resolved"] == 0 - assert meta["bookmarks_unresolved"] == 1 - assert meta["headings_found"] == 1 - assert meta["heading_source"] == "font_analysis" - - @pytest.mark.asyncio - async def test_convert_local_skips_images_stacked_on_same_bbox(self): - """Two image XObjects at the same spot must render (and save) only once.""" - parser = PDFParser() - stacked = {"x0": 0.0, "top": 0.1, "x1": 595.0, "bottom": 841.9} - page = _FakePage("Page one") - page.images = [dict(stacked), dict(stacked)] - fake_pdf = SimpleNamespace(pages=[page]) - fake_pdfplumber = SimpleNamespace(open=lambda _path: nullcontext(fake_pdf)) - storage = MagicMock() - storage.save_image.return_value = Path("/media/dummy/images/page1_img1.png") - storage.media_dir = Path("/media") - - with ( - patch("openviking.parse.parsers.pdf.lazy_import", return_value=fake_pdfplumber), - patch.object(parser, "_extract_bookmarks", return_value=[]), - patch.object(parser, "_detect_headings_by_font", return_value=[]), - patch.object(parser, "_extract_image_from_page", return_value=b"png") as extract, - ): - markdown, meta = await parser._convert_local( - "dummy.pdf", storage=storage, resource_name="dummy" - ) - - # The duplicate is dropped before rendering, so it never costs a render. - assert extract.call_count == 1 - assert storage.save_image.call_count == 1 - assert meta["images_extracted"] == 1 - assert meta["images_deduplicated"] == 1 - assert markdown.count("![Page 1 Image") == 1 - - @pytest.mark.asyncio - async def test_convert_local_skips_images_rendering_to_same_bytes(self): - """Distinct bboxes that still render identically are caught by the hash.""" - parser = PDFParser() - page = _FakePage("Page one") - page.images = [ - {"x0": 0.0, "top": 0.0, "x1": 100.0, "bottom": 100.0}, - {"x0": 5.0, "top": 5.0, "x1": 105.0, "bottom": 105.0}, - ] - fake_pdf = SimpleNamespace(pages=[page]) - fake_pdfplumber = SimpleNamespace(open=lambda _path: nullcontext(fake_pdf)) - storage = MagicMock() - storage.save_image.return_value = Path("/media/dummy/images/page1_img1.png") - storage.media_dir = Path("/media") - - with ( - patch("openviking.parse.parsers.pdf.lazy_import", return_value=fake_pdfplumber), - patch.object(parser, "_extract_bookmarks", return_value=[]), - patch.object(parser, "_detect_headings_by_font", return_value=[]), - patch.object(parser, "_extract_image_from_page", return_value=b"png") as extract, - ): - _markdown, meta = await parser._convert_local( - "dummy.pdf", storage=storage, resource_name="dummy" - ) - - # Both are rendered (bboxes differ), but only one is saved. - assert extract.call_count == 2 - assert storage.save_image.call_count == 1 - assert meta["images_extracted"] == 1 - assert meta["images_deduplicated"] == 1 - - @pytest.mark.asyncio - async def test_convert_local_keeps_distinct_images_and_repeats_across_pages(self): - """Dedup is per-page: a logo on every page survives on every page.""" - parser = PDFParser() - logo = {"x0": 0.0, "top": 0.0, "x1": 50.0, "bottom": 50.0} - figure = {"x0": 0.0, "top": 200.0, "x1": 300.0, "bottom": 400.0} - page1, page2 = _FakePage("Page one"), _FakePage("Page two") - page1.images = [dict(logo), dict(figure)] - page2.images = [dict(logo)] - fake_pdf = SimpleNamespace(pages=[page1, page2]) - fake_pdfplumber = SimpleNamespace(open=lambda _path: nullcontext(fake_pdf)) - storage = MagicMock() - storage.save_image.return_value = Path("/media/dummy/images/img.png") - storage.media_dir = Path("/media") - - renders = { - (0.0, 0.0, 50.0, 50.0): b"logo-png", - (0.0, 200.0, 300.0, 400.0): b"figure-png", - } - - with ( - patch("openviking.parse.parsers.pdf.lazy_import", return_value=fake_pdfplumber), - patch.object(parser, "_extract_bookmarks", return_value=[]), - patch.object(parser, "_detect_headings_by_font", return_value=[]), - patch.object( - parser, - "_extract_image_from_page", - side_effect=lambda _page, img: renders[ - (img["x0"], img["top"], img["x1"], img["bottom"]) - ], - ), - ): - _markdown, meta = await parser._convert_local( - "dummy.pdf", storage=storage, resource_name="dummy" - ) - - assert meta["images_extracted"] == 3 - assert meta["images_deduplicated"] == 0 - - @pytest.mark.asyncio - async def test_convert_local_closes_page_after_each_page(self): - parser = PDFParser() - pages = [_FakePage("Page one"), _FakePage("Page two")] - fake_pdf = SimpleNamespace(pages=pages) - fake_pdfplumber = SimpleNamespace(open=lambda _path: nullcontext(fake_pdf)) - - with ( - patch("openviking.parse.parsers.pdf.lazy_import", return_value=fake_pdfplumber), - patch.object(parser, "_extract_bookmarks", return_value=[]), - patch.object(parser, "_detect_headings_by_font", return_value=[]), - ): - await parser._convert_local("dummy.pdf", storage=MagicMock(), resource_name="dummy") - - assert [page.close_count for page in pages] == [1, 1] - assert [page.flush_count for page in pages] == [1, 1] - - def test_detect_headings_by_font_closes_pages(self): - parser = PDFParser() - - def chars(text: str, size: float, top: float) -> list[dict]: - return [ - {"text": char, "size": size, "top": top, "x0": idx} for idx, char in enumerate(text) - ] - - pages = [ - _FakeFontPage( - page_number=0, - chars=chars("Body text repeated", 10, 500) + chars("Heading", 14, 100), - ), - _FakeFontPage( - page_number=1, - chars=chars("Another heading", 14, 120), - ), - _FakeFontPage( - page_number=2, - chars=chars("Plain body", 10, 500), - ), - _FakeFontPage( - page_number=3, - chars=chars("More body", 10, 500), - ), - ] - fake_pdf = SimpleNamespace(pages=pages) - - headings = parser._detect_headings_by_font(fake_pdf) - - assert [heading["title"] for heading in headings] == ["Heading", "Another heading"] - assert [page.close_count for page in pages] == [2, 1, 1, 1] - assert [page.flush_count for page in pages] == [2, 1, 1, 1] - - -class TestExtractImages: - """Test PDF XObject image extraction.""" - - def setup_method(self): - self.parser = PDFParser() - - def test_extract_image_renders_cropped_bbox_as_png(self): - page = _FakeImagePage(image_bytes=b"rendered-png") - - image_data = self.parser._extract_image_from_page( - page, - {"x0": 10, "top": 20, "x1": 40, "bottom": 60}, - ) + def __init__(self) -> None: + self.closed = False - assert image_data == b"rendered-png" - assert page.crop_calls == [(10, 20, 40, 60)] - assert page.cropped_page.resolution_calls == [self.parser.config.image_resolution] + def close(self) -> None: + self.closed = True - def test_extract_image_clamps_bbox_to_page_bounds(self): - page = _FakeImagePage() + page = FakePage() - self.parser._extract_image_from_page( - page, - {"x0": -5, "top": -10, "x1": 150, "bottom": 250}, - ) + class FakePDF: + pages = [page] - assert page.crop_calls == [(0, 0, page.width, page.height)] + def __enter__(self): + return self - def test_extract_image_returns_none_for_zero_area_bbox(self): - page = _FakeImagePage() + def __exit__(self, *_args): + return None - assert ( - self.parser._extract_image_from_page( - page, - {"x0": 30, "top": 40, "x1": 30, "bottom": 60}, - ) - is None - ) - assert page.crop_calls == [] + monkeypatch.setitem( + sys.modules, + "pdfplumber", + SimpleNamespace(open=lambda _path: FakePDF()), + ) + parser = PDFParser() + monkeypatch.setattr(parser, "_extract_image_from_page", lambda *_args: b"png") + monkeypatch.setattr("openviking.parse.parsers.pdf.is_valid_image", lambda *_args: True) + storage = _FakeStorage(tmp_path / "media") - def test_extract_image_returns_none_when_crop_fails(self): - page = MagicMock(width=100, height=200) - page.crop.side_effect = RuntimeError("crop failed") + result = parser._extract_images_sync(tmp_path / "paper.pdf", storage, "paper") - assert ( - self.parser._extract_image_from_page( - page, - {"x0": 10, "top": 20, "x1": 40, "bottom": 60}, - ) - is None - ) + assert result.pages == {1: ["![Page 1 Image 1](paper/images/page1_img1.png)"]} + assert result.images_extracted == 1 + assert result.images_deduplicated == 1 + assert page.closed is True diff --git a/tests/test_config_loader.py b/tests/test_config_loader.py index 9cf26591c0..bae51b50d3 100644 --- a/tests/test_config_loader.py +++ b/tests/test_config_loader.py @@ -198,7 +198,7 @@ def test_generic_code_hosting_domains_load_from_config(): assert config.code_hosting_domains == ["git.generic.example.com"] -def test_openviking_config_rejects_unknown_nested_parser_section(monkeypatch): +def test_openviking_config_rejects_removed_excel_parser_section(monkeypatch): monkeypatch.setenv(OPENVIKING_CONFIG_ENV, "/tmp/codex-no-config.json") from openviking_cli.utils.config.open_viking_config import ( @@ -206,7 +206,7 @@ def test_openviking_config_rejects_unknown_nested_parser_section(monkeypatch): OpenVikingConfigSingleton, ) - with pytest.raises(ValueError, match="markdown"): + with pytest.raises(ValueError, match="excel"): OpenVikingConfig.from_dict( { "embedding": { @@ -216,7 +216,7 @@ def test_openviking_config_rejects_unknown_nested_parser_section(monkeypatch): "model": "text-embedding-3-small", } }, - "parsers": {"markdwon": {}}, + "parsers": {"excel": {}}, } ) diff --git a/uv.lock b/uv.lock index 3cfb2dba97..c7a0e1ae32 100644 --- a/uv.lock +++ b/uv.lock @@ -1365,6 +1365,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f9/0b/de6f54d4a8bedfe8645c41497f3c18d749f0bd3218170c667bf4b81d0cdd/filelock-3.25.0-py3-none-any.whl", hash = "sha256:5ccf8069f7948f494968fc0713c10e5c182a9c9d9eef3a636307a20c2490f047", size = 26427, upload-time = "2026-03-01T15:08:44.593Z" }, ] +[[package]] +name = "firecrawl-anydoc" +version = "0.1.8" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e9/58/94bdc78509da5d2bec4761cfc6e79061d35562d4397b504aec59e29b548b/firecrawl_anydoc-0.1.8.tar.gz", hash = "sha256:342289a9e7fcc46a312919547b475827f202f6824c2acf05e334fc1ed386e77e", size = 197922, upload-time = "2026-08-10T23:39:54.336Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/97/61/abd70c4215e2804c8dad88c2abe8685fa84f4d2496ce7fc17d7f7a1cc21d/firecrawl_anydoc-0.1.8-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:8a7218553f691f112a2d71f2c08c3ef87de3ce4d15fa82ffa782f493a307a284", size = 3447457, upload-time = "2026-08-10T23:39:39.832Z" }, + { url = "https://files.pythonhosted.org/packages/cd/c1/ab54eb4ca0a131017941964219fbf80873059b4261b046f8dda1bdeb858c/firecrawl_anydoc-0.1.8-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:8920b0d303ae5ad34bc5e580b25043eaf107df73e4f8f2f02282bfad7f19845a", size = 3273654, upload-time = "2026-08-10T23:39:42.596Z" }, + { url = "https://files.pythonhosted.org/packages/af/10/9216bb6e65d027aac7c4c856ad921684337c105f69db77703059afd1ec1e/firecrawl_anydoc-0.1.8-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cfe87061151cb42dd52960eb84f4ec6249a220131e61b498bba1248b86ef51cc", size = 3313860, upload-time = "2026-08-10T23:39:44.664Z" }, + { url = "https://files.pythonhosted.org/packages/b8/23/0d4fb73798b6554a63f38468e62ce13f1d411ef3dd879efc4c89f535ee44/firecrawl_anydoc-0.1.8-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d3ba80a81f1831b5cdfcc14678e3ed13078587710fb3f5556f193dbcaa901070", size = 3540091, upload-time = "2026-08-10T23:39:46.642Z" }, + { url = "https://files.pythonhosted.org/packages/17/f6/438c158410c0f696dbb29e4f0b83cbb6cd89511792fd97917aaa4f2cc76e/firecrawl_anydoc-0.1.8-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:07e4b9444cc979f2eb35289c8d28d738474209824a29d07081af25cd6a65bf9b", size = 3492643, upload-time = "2026-08-10T23:39:48.579Z" }, + { url = "https://files.pythonhosted.org/packages/85/ba/bd6b8502093ddcc8fe170fdc20a37009dc5e7a6ae55551ede69a9639ecf8/firecrawl_anydoc-0.1.8-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4b7d79b8f7c158a67232982dc1883c07114d95f69d7aedc902d0b113a5546b16", size = 3780576, upload-time = "2026-08-10T23:39:50.458Z" }, + { url = "https://files.pythonhosted.org/packages/df/91/743986265ae347f0380a21d60378155575f4d724465c77c7e4781923df57/firecrawl_anydoc-0.1.8-cp310-abi3-win_amd64.whl", hash = "sha256:bf755eb80439f0a1540b298648919f19751c16f942a7376ee68dea5318e3f2a3", size = 3632712, upload-time = "2026-08-10T23:39:52.561Z" }, +] + [[package]] name = "frozenlist" version = "1.8.0" @@ -3415,15 +3430,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/be/9c/92789c596b8df838baa98fa71844d84283302f7604ed565dafe5a6b5041a/oauthlib-3.3.1-py3-none-any.whl", hash = "sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1", size = 160065, upload-time = "2025-06-19T22:48:06.508Z" }, ] -[[package]] -name = "olefile" -version = "0.47" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/69/1b/077b508e3e500e1629d366249c3ccb32f95e50258b231705c09e3c7a4366/olefile-0.47.zip", hash = "sha256:599383381a0bf3dfbd932ca0ca6515acd174ed48870cbf7fee123d698c192c1c", size = 112240, upload-time = "2023-12-01T16:22:53.025Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/17/d3/b64c356a907242d719fc668b71befd73324e47ab46c8ebbbede252c154b2/olefile-0.47-py2.py3-none-any.whl", hash = "sha256:543c7da2a7adadf21214938bb79c83ea12b473a4b6ee4ad4bf854e7715e13d1f", size = 114565, upload-time = "2023-12-01T16:22:51.518Z" }, -] - [[package]] name = "openai" version = "2.24.0" @@ -3646,9 +3652,9 @@ dependencies = [ { name = "charset-normalizer" }, { name = "cryptography" }, { name = "defusedxml" }, - { name = "ebooklib" }, { name = "fastapi" }, { name = "feedparser" }, + { name = "firecrawl-anydoc" }, { name = "grep-ast" }, { name = "httpx" }, { name = "jinja2" }, @@ -3657,9 +3663,7 @@ dependencies = [ { name = "litellm" }, { name = "loguru" }, { name = "mcp" }, - { name = "olefile" }, { name = "openai" }, - { name = "openpyxl" }, { name = "opentelemetry-api" }, { name = "opentelemetry-exporter-otlp-proto-grpc" }, { name = "opentelemetry-exporter-otlp-proto-http" }, @@ -3667,13 +3671,11 @@ dependencies = [ { name = "opentelemetry-sdk" }, { name = "openviking-sdk" }, { name = "pathspec" }, - { name = "pdfminer-six" }, + { name = "pdf-inspector" }, { name = "pdfplumber" }, { name = "protobuf" }, { name = "pydantic" }, - { name = "python-docx" }, { name = "python-multipart" }, - { name = "python-pptx" }, { name = "pyyaml" }, { name = "requests" }, { name = "scrapy" }, @@ -3697,7 +3699,6 @@ dependencies = [ { name = "uvicorn" }, { name = "volcengine" }, { name = "volcengine-python-sdk", extra = ["ark"] }, - { name = "xlrd" }, { name = "xxhash" }, ] @@ -3794,13 +3795,17 @@ test = [ { name = "boto3" }, { name = "datasets" }, { name = "diff-match-patch" }, + { name = "ebooklib" }, { name = "hvac" }, + { name = "openpyxl" }, { name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "pandas", version = "3.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "pytest" }, { name = "pytest-asyncio" }, { name = "pytest-cov" }, { name = "pytest-xdist" }, + { name = "python-docx" }, + { name = "python-pptx" }, { name = "ragas" }, ] @@ -3829,9 +3834,10 @@ requires-dist = [ { name = "defusedxml", specifier = ">=0.7.1" }, { name = "diff-match-patch", marker = "extra == 'test'", specifier = ">=20200713" }, { name = "dingtalk-stream", marker = "extra == 'bot'", specifier = ">=0.4.0" }, - { name = "ebooklib", specifier = ">=0.18.0" }, + { name = "ebooklib", marker = "extra == 'test'", specifier = ">=0.18.0" }, { name = "fastapi", specifier = ">=0.128.0" }, { name = "feedparser", specifier = ">=6.0.0" }, + { name = "firecrawl-anydoc", specifier = ">=0.1.8,<0.2" }, { name = "fusepy", marker = "extra == 'bot'", specifier = ">=3.0.1" }, { name = "google-genai", marker = "extra == 'gemini'", specifier = ">=1.0.0" }, { name = "google-genai", marker = "extra == 'gemini-async'", specifier = ">=1.0.0" }, @@ -3858,10 +3864,9 @@ requires-dist = [ { name = "msgpack", marker = "extra == 'bot'", specifier = ">=1.0.8" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.0.0" }, { name = "myst-parser", marker = "extra == 'doc'", specifier = ">=2.0.0" }, - { name = "olefile", specifier = ">=0.47" }, { name = "openai", specifier = ">=1.0.0" }, { name = "opencode-ai", marker = "extra == 'bot'", specifier = ">=0.1.0a0" }, - { name = "openpyxl", specifier = ">=3.0.0" }, + { name = "openpyxl", marker = "extra == 'test'", specifier = ">=3.0.0" }, { name = "opensandbox", marker = "extra == 'bot'", specifier = ">=0.1.0" }, { name = "opensandbox-server", marker = "extra == 'bot'", specifier = ">=0.1.0" }, { name = "opentelemetry-api", specifier = ">=1.14" }, @@ -3874,7 +3879,7 @@ requires-dist = [ { name = "pandas", marker = "extra == 'eval'", specifier = ">=2.0.0" }, { name = "pandas", marker = "extra == 'test'", specifier = ">=2.0.0" }, { name = "pathspec", specifier = ">=1.1.1" }, - { name = "pdfminer-six", specifier = ">=20251230" }, + { name = "pdf-inspector", specifier = ">=1.14.1,<2" }, { name = "pdfplumber", specifier = ">=0.10.0" }, { name = "prompt-toolkit", marker = "extra == 'bot'", specifier = ">=3.0.0" }, { name = "protobuf", specifier = ">=6.33.5" }, @@ -3887,13 +3892,13 @@ requires-dist = [ { name = "pytest-asyncio", marker = "extra == 'test'", specifier = ">=0.21.0" }, { name = "pytest-cov", marker = "extra == 'test'", specifier = ">=4.0.0" }, { name = "pytest-xdist", marker = "extra == 'test'", specifier = ">=3.5.0" }, - { name = "python-docx", specifier = ">=1.0.0" }, + { name = "python-docx", marker = "extra == 'test'", specifier = ">=1.0.0" }, { name = "python-engineio", marker = "extra == 'bot'", specifier = ">=4.13.2" }, { name = "python-jose", extras = ["cryptography"], marker = "extra == 'auth'", specifier = ">=3.3.0" }, { name = "python-jose", extras = ["cryptography"], marker = "extra == 'bot'", specifier = ">=3.3.0" }, { name = "python-ldap", marker = "extra == 'auth'", specifier = ">=3.4.0" }, { name = "python-multipart", specifier = ">=0.0.31" }, - { name = "python-pptx", specifier = ">=1.0.0" }, + { name = "python-pptx", marker = "extra == 'test'", specifier = ">=1.0.0" }, { name = "python-socketio", marker = "extra == 'bot'", specifier = ">=5.16.2" }, { name = "python-socks", extras = ["asyncio"], marker = "extra == 'bot'", specifier = ">=2.4.0" }, { name = "python-telegram-bot", extras = ["socks"], marker = "extra == 'bot'", specifier = ">=21.0" }, @@ -3938,10 +3943,9 @@ requires-dist = [ { name = "websocket-client", marker = "extra == 'bot'", specifier = ">=1.6.0" }, { name = "websockets", marker = "extra == 'bot'", specifier = ">=12.0" }, { name = "wheel", marker = "extra == 'build'" }, - { name = "xlrd", specifier = ">=2.0.1" }, { name = "xxhash", specifier = ">=3.0.0" }, ] -provides-extras = ["test", "auth", "dev", "doc", "eval", "gemini", "gemini-async", "ocr", "build", "bot", "benchmark", "langchain", "langgraph", "local-embed"] +provides-extras = ["test", "auth", "dev", "doc", "eval", "gemini", "gemini-async", "ocr", "build", "bot", "benchmark", "local-embed"] [package.metadata.requires-dev] dev = [{ name = "pytest", specifier = ">=9.0.2" }] @@ -4267,6 +4271,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/d9/7fb5aa316bc299258e68c73ba3bddbc499654a07f151cba08f6153988714/pathspec-1.1.1-py3-none-any.whl", hash = "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189", size = 57328, upload-time = "2026-04-27T01:46:07.06Z" }, ] +[[package]] +name = "pdf-inspector" +version = "1.14.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cc/7d/29ac7865f4c7332d28638d08b73aa6b7bf2f1325c383094bb6717a1a816d/pdf_inspector-1.14.1.tar.gz", hash = "sha256:e9d0fc1669ca77950d28488c8450d1aa28018e839ea260308d87253dfb8bec03", size = 1559937, upload-time = "2026-08-11T21:22:53.948Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/96/47/7ddacbcac7a04ec60014e2a03b943c4643a8bd794ed67588f5ebadfbeb15/pdf_inspector-1.14.1-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:b2159efa28f132df0442129a9b3eaf70c3f9915eb8202efcaaf759d85a01993f", size = 2855526, upload-time = "2026-08-11T21:22:44.66Z" }, + { url = "https://files.pythonhosted.org/packages/98/24/00594b8ae250c0b8a87631a86067896416920879c43c8cdcdfeb50586d49/pdf_inspector-1.14.1-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:54cb32691640e5684832d21fbb58d2b30c068cbc055e0157da37774c1e8b9218", size = 2767918, upload-time = "2026-08-11T21:22:46.768Z" }, + { url = "https://files.pythonhosted.org/packages/cd/36/5b332c80858407fb19b34e6f18f6c99022d04324caf98b8aaf4547044871/pdf_inspector-1.14.1-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:501b8a6ffcd2c16ea74d71b9d7d70503c945c7d38c81151121790fd278021f0b", size = 2942959, upload-time = "2026-08-11T21:22:48.771Z" }, + { url = "https://files.pythonhosted.org/packages/37/cc/589c355fed7b50098ddb9d09116c81d966c7b4ff78741183e600c1b4de6f/pdf_inspector-1.14.1-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:62e761a32ffd9c2b0c803dbefa2b0ad6f98bc26a75847571d99c3f7a65a7cea3", size = 3044612, upload-time = "2026-08-11T21:22:50.571Z" }, + { url = "https://files.pythonhosted.org/packages/24/7c/4c84916a97cfd85d2e72dc3aaf6988cd19aff6e050d6a1b79d20f97e4566/pdf_inspector-1.14.1-cp38-abi3-win_amd64.whl", hash = "sha256:d4a2455c316e15eac745c3836f72b641e1c33207afb84c250abff2843209e527", size = 2690743, upload-time = "2026-08-11T21:22:52.407Z" }, +] + [[package]] name = "pdfminer-six" version = "20251230" @@ -7053,15 +7070,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a4/f5/10b68b7b1544245097b2a1b8238f66f2fc6dcaeb24ba5d917f52bd2eed4f/wsproto-1.3.2-py3-none-any.whl", hash = "sha256:61eea322cdf56e8cc904bd3ad7573359a242ba65688716b0710a5eb12beab584", size = 24405, upload-time = "2025-11-20T18:18:00.454Z" }, ] -[[package]] -name = "xlrd" -version = "2.0.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/07/5a/377161c2d3538d1990d7af382c79f3b2372e880b65de21b01b1a2b78691e/xlrd-2.0.2.tar.gz", hash = "sha256:08b5e25de58f21ce71dc7db3b3b8106c1fa776f3024c54e45b45b374e89234c9", size = 100167, upload-time = "2025-06-14T08:46:39.039Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1a/62/c8d562e7766786ba6587d09c5a8ba9f718ed3fa8af7f4553e8f91c36f302/xlrd-2.0.2-py2.py3-none-any.whl", hash = "sha256:ea762c3d29f4cca48d82df517b6d89fbce4db3107f9d78713e48cd321d5c9aa9", size = 96555, upload-time = "2025-06-14T08:46:37.766Z" }, -] - [[package]] name = "xlsxwriter" version = "3.2.9"