diff --git a/.env.example b/.env.example index e18ae5f8..4d8c18d2 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,7 @@ # Production and shared environments use the fail-closed authentication mode. NEWSDOM_AUTH_MODE=required +# Per-process non-waiting parser admission budget. Scale replicas separately. +NEWSDOM_MAX_CONCURRENT_PARSES=1 NEWSDOM_RUNTIME_PROFILE=production # Generate a high-entropy token in the deployment secret store. Never commit it. NEWSDOM_API_TOKEN= diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 9a87b80d..c0879e0b 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -2,54 +2,112 @@ ## Runtime shape -`newsdom-api` is a small service-oriented Python application with a -thin FastAPI entrypoint and explicit separation between request -orchestration, MinerU process execution, and DOM normalization. +`newsdom-api` is a small service-oriented Python application with a thin +FastAPI entrypoint and explicit separation between request security, +process-local admission, MinerU process execution, and DOM normalization. The +same application can run as a standalone service, a sidecar, or a bounded MSA +module behind an API gateway. ## Primary modules -- `src/newsdom_api/main.py` exposes `/health` and `/parse` - through FastAPI. -- `src/newsdom_api/service.py` orchestrates PDF parsing, - temporary files, and response construction. -- `src/newsdom_api/mineru_runner.py` shells out to the MinerU CLI, - collects JSON outputs, and translates runtime or incomplete-output - failures into typed sanitized exceptions. -- `src/newsdom_api/dom_builder.py` converts MinerU `content_list` - blocks plus page model metadata into the canonical NewsDOM response - model. +- `src/newsdom_api/main.py` exposes `/health`, `/ready`, and `/parse` through + FastAPI. Its outer middleware authenticates and admits parser work before the + multipart body is read. +- `src/newsdom_api/config.py` snapshots immutable runtime authentication, + profile, and per-process capacity settings during application creation. +- `src/newsdom_api/admission.py` owns the process-local non-waiting parser lease + pool. `ParseAdmissionLimiter` uses `threading.BoundedSemaphore` so excess + release is treated as a programming error rather than silently inflating + capacity. +- `src/newsdom_api/service.py` orchestrates PDF parsing and response + construction. +- `src/newsdom_api/mineru_runner.py` shells out to the MinerU CLI, collects JSON + outputs, and translates runtime or incomplete-output failures into typed + sanitized exceptions. +- `src/newsdom_api/dom_builder.py` converts MinerU `content_list` blocks plus + page model metadata into the canonical NewsDOM response model. - `src/newsdom_api/schemas.py` defines the public response schema. -- `src/newsdom_api/synthetic.py` and - `src/newsdom_api/equivalence.py` support synthetic fixture - generation and structural comparisons. +- `src/newsdom_api/synthetic.py` and `src/newsdom_api/equivalence.py` support + synthetic fixture generation and structural comparisons. ## Request flow -1. `src/newsdom_api/main.py` receives an uploaded PDF. -2. `src/newsdom_api/service.py` writes the upload to a temporary - workspace and calls MinerU. -3. `src/newsdom_api/mineru_runner.py` resolves the executable, runs - the OCR pipeline, loads generated JSON artifacts, and raises typed - sanitized errors for runtime-unavailable or incomplete-output cases. -4. `src/newsdom_api/dom_builder.py` normalizes OCR blocks into the - canonical response while preserving page-aware structure from - MinerU model metadata. -5. FastAPI returns typed JSON from `src/newsdom_api/schemas.py` and - maps MinerU runtime failures to 503 and incomplete output to 502. +```mermaid +flowchart LR + C[Client] --> A[Raw bearer authentication] + A -->|invalid| U[401 or configuration 503] + A -->|valid| L[Process-local non-waiting lease] + L -->|saturated| R[429 Too Many Requests\nRetry-After: 1] + L -->|admitted| M[Multipart body and upload limits] + M --> P[PDF structural validation] + P --> T[Temporary file] + T --> X[MinerU execution] + X --> D[Canonical DOM normalization] + D --> J[Typed JSON response] + J --> F[Lease release] + X -->|error or cancellation| F +``` + +1. `src/newsdom_api/main.py` validates the raw Authorization header without + decoding duplicate or non-ASCII credentials into a lossy string. +2. An authenticated `POST /parse` attempts one lease from + `ParseAdmissionLimiter` **before the multipart body** is parsed. The attempt + never waits. Saturation returns `429 Too Many Requests`, `Retry-After: 1`, + and non-cacheable response headers. +3. The route validates MIME type, form values, declared and streamed size, and + PDF structure before parser execution. +4. `src/newsdom_api/service.py` and `src/newsdom_api/mineru_runner.py` execute + MinerU under the configured runtime and error boundaries. +5. `src/newsdom_api/dom_builder.py` normalizes OCR blocks into the canonical + response while preserving page-aware structure from MinerU model metadata. +6. The outer `finally` boundary releases the parser lease after success, + validation failure, backend failure, request cancellation, or another + exception. +7. FastAPI returns typed JSON from `src/newsdom_api/schemas.py` and maps MinerU + runtime failures to 503 and incomplete output to 502. + +## Capacity model + +`NEWSDOM_MAX_CONCURRENT_PARSES` is immutable after application creation, accepts +`1..128`, and defaults to `1`. The budget is **per process**, not global. A +single-process replica configured with `N` can execute at most `N` MinerU jobs +at once. A deployment with `R` replicas and one serving process in each has a +nominal upper bound of `R × N`; additional gateway, scheduler, CPU, memory, +VRAM, and storage constraints can reduce effective throughput. + +The application intentionally does not wait for a parser slot. An internal +queue would already retain request bodies and open connections, hide overload +from the orchestrator, and make memory consumption proportional to demand. +The fixed 429 contract instead keeps queue ownership at a gateway or durable +job service that can implement tenant policy, persistence, cancellation, and +bounded retry semantics. + +## Liveness and readiness + +`GET /health` proves only that the web process is live. `GET /ready` proves that +fail-closed authentication configuration and an approved parser backend are +available. A discoverable legacy MinerU executable alone is not readiness. Parser saturation does not make a process unready: readiness is a +routing eligibility signal, whereas the admission response is instantaneous +load feedback for each request. A future durable asynchronous job API may add +queue-depth readiness policy without changing this synchronous endpoint's +non-waiting contract. ## Supporting systems -- `tests/fixtures` holds synthetic PDFs, JSON baselines, and - provenance notes; private reference inputs stay out of git. +- `tests/fixtures` holds synthetic PDFs, JSON baselines, and provenance notes; + private reference inputs stay out of git. - `manual/` is the published user manual rendered by MkDocs. -- `.github/workflows/` encodes CI, security scanning, Pages, - release, and image-delivery policy. -- `scripts/release/` builds release manifests and exports GitHub attestation bundles. +- `.github/workflows/` encodes CI, security scanning, Pages, release, and + image-delivery policy. +- `scripts/release/` builds release manifests and exports GitHub attestation + bundles. +- `docs/doctoring/bounded-parser-admission.md` records the resource-exhaustion, + HTTP, caching, implementation, rollback, and APA 7 evidence behind the + admission boundary. ## Delivery boundaries -- `develop` is the integration line for normal feature, fix, - and chore work. +- `develop` is the integration line for normal feature, fix, and chore work. - `main` is the stable release line that receives tagged releases. -- The service is production-grade only when code, docs, workflows, - and release evidence agree. +- The service is production-grade only when code, docs, workflows, and release + evidence agree. diff --git a/CHANGELOG.md b/CHANGELOG.md index f03b174f..1528cba0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,10 +16,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 > acceptance are aligned for an actual 0.3.0 publication. ### Changed + - `/parse`를 언어 선택형 파서로 일반화: MinerU `-l japan`/`-m ocr` 하드코딩을 제거하고 optional form 필드 `language`(MinerU 3.4.4 공식 기본 `ch`, 공개 언어군/alias 검증)와 `mode`(`auto`/`ocr`/`txt`, 기본 `auto`)로 파라미터화. `mode=auto`는 born-digital PDF가 강제 OCR을 건너뛰도록 함. 기존 입력 `language=japan&mode=ocr`는 공식 규약대로 `ch`/`ocr`로 정규화됨. - OpenAPI 제목/설명, README, `ArticleNode.headline` 문서를 일반 문서용 (section heading) 표현으로 재구성하여 특정 언어/신문 가정을 소비자에게 노출하지 않도록 함. 응답 스키마 필드는 하위 호환을 위해 변경하지 않음. ### Added +- Added immutable `NEWSDOM_MAX_CONCURRENT_PARSES` process admission with a conservative default of one. Authentication remains first; an authenticated request must acquire a non-waiting lease before the multipart body, temporary-file allocation, PDF validation, or MinerU. Saturation returns fixed `429 Too Many Requests`, `Retry-After: 1`, and no-store cache headers, while every success, error, exception, and cancellation path releases its lease. Docker Compose and Kubernetes examples publish the explicit per-process value and explain replica capacity. - [CLI] 단일 NewsDOM JSON 파일을 페이지 단위로 분리하는 `tools/split_dom.py` 도구를 추가했습니다. - [CLI] NewsDOM JSON 파일의 모든 텍스트 내용을 마스킹하여 익명화하는 `tools/anonymize_dom.py` 도구를 추가했습니다. - `/parse`에 기본 필수 bearer 인증 경계를 추가했습니다. `NEWSDOM_AUTH_MODE=required`, `NEWSDOM_RUNTIME_PROFILE=production`, `NEWSDOM_API_TOKEN`을 명시해야 하며, 인증 비활성화는 격리된 development 프로필에서만 허용됩니다. `/health`는 liveness 전용으로 미인증 상태를 유지하고 `/ready`가 인증 설정과 MinerU 가용성을 함께 검증합니다. @@ -27,8 +29,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [CLI] 파싱된 NewsDOM JSON에서 순수 텍스트 데이터를 추출하여 텍스트 파일 또는 stdout으로 출력하는 `tools/extract_text.py` 도구를 추가했습니다. ### Security + +- Bounded active parser work prevents one process from starting unbounded MinerU jobs under burst traffic. Excess requests fail before body reads with the RFC 6585 429 contract and RFC 9111 no-store semantics; gateway-level tenant quotas and durable job queues remain separate controls. - `/parse` authentication is now immutable per application instance and fails closed before multipart body parsing when required configuration is missing. Hostile missing, invalid, Unicode, oversized, and duplicated Authorization headers return one non-sensitive response. -- Added unauthenticated `/ready` traffic readiness that combines authentication configuration with MinerU executable availability while `/health` remains liveness-only. +- Added unauthenticated `/ready` traffic readiness that combines authentication configuration with approved-parser availability; a legacy MinerU executable alone remains fail-closed while `/health` remains liveness-only. - Hardened the Kubernetes deployment example with a restricted namespace policy, explicit non-root UID/GID, `RuntimeDefault` seccomp, disabled privilege escalation, dropped Linux capabilities, a read-only root filesystem, and bounded writable runtime volumes. - 전역 500 에러 응답에도 표준 보안 헤더를 적용하여 예외 경로에서 header 누락을 방지 - MinerU subprocess argv 생성 시 `-`로 시작하는 option-like 인자를 거부하여 argument injection 위험을 낮춤 @@ -64,7 +68,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- Improved OpenAPI metadata for the `/parse` endpoint by documenting 415, 502, and 503 error responses. +- Improved OpenAPI metadata for the `/parse` endpoint by documenting 415, 429, 502, and 503 error responses. ## [0.2.0] - 2026-04-24 @@ -108,7 +112,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Synthetic newspaper fixture generation and structural equivalence checks - Protected-branch CI, security gates, release provenance workflow, and Git Flow documentation -[Unreleased]: https://github.com/Seongho-Bae/newsdom-api/compare/v0.2.0...HEAD -[0.2.0]: https://github.com/Seongho-Bae/newsdom-api/compare/v0.1.1...v0.2.0 -[0.1.1]: https://github.com/Seongho-Bae/newsdom-api/compare/v0.1.0...v0.1.1 -[0.1.0]: https://github.com/Seongho-Bae/newsdom-api/releases/tag/v0.1.0 +[Unreleased]: https://github.com/ContextualWisdomLab/newsdom-api/compare/v0.2.0...HEAD +[0.2.0]: https://github.com/ContextualWisdomLab/newsdom-api/compare/v0.1.1...v0.2.0 +[0.1.1]: https://github.com/ContextualWisdomLab/newsdom-api/compare/v0.1.0...v0.1.1 +[0.1.0]: https://github.com/ContextualWisdomLab/newsdom-api/releases/tag/v0.1.0 diff --git a/README.md b/README.md index 4b5c1674..a7c68b9b 100644 --- a/README.md +++ b/README.md @@ -1,197 +1,141 @@ # NewsDOM API -[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/Seongho-Bae/newsdom-api/badge)](https://securityscorecards.dev/viewer/?uri=github.com/Seongho-Bae/newsdom-api) +[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/ContextualWisdomLab/newsdom-api/badge)](https://securityscorecards.dev/viewer/?uri=github.com/ContextualWisdomLab/newsdom-api) +[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/ContextualWisdomLab/newsdom-api) -NewsDOM API is a language-agnostic PDF-to-DOM sidecar. It converts any PDF -into a canonical JSON document tree — pages, sections, headings, body -blocks, images, captions, and bounding boxes — using MinerU. It runs as a -standalone FastAPI service and is designed to be embedded as a git -submodule / sidecar in a larger system. +**Turn PDFs into a stable, structured document tree that downstream systems can inspect instead of reverse-engineering parser output.** -## Features +NewsDOM API owns the HTTP, safety, normalization, and evidence boundary for PDF-to-DOM processing. It accepts a PDF, applies bounded admission and fail-closed request controls, and returns canonical JSON for pages, sections, headings, body blocks, images, captions, bounding boxes, and quality metadata. -- Primary engine: `MinerU` pipeline backend -- Service wrapper: FastAPI -- Output: canonical JSON with pages, sections, section headings, body - blocks, images, captions, bounding boxes, and quality metadata -- Language-selectable: `language` defaults to MinerU's multilingual `ch` model; - official language families and compatibility aliases such as `japan` remain - available -- Parsing `mode` defaults to `auto` so born-digital text PDFs skip forced OCR -- Fail-closed bearer authentication on `/parse`; separate unauthenticated `/health` liveness and `/ready` traffic-readiness probes +The service is intentionally usable as a standalone FastAPI sidecar. A host application does not need to adopt NewsDOM's internal parser process, storage layout, or deployment model; it consumes the published HTTP/schema contract. -## Quickstart +> **Commercial parser status:** the current source still contains a legacy MinerU adapter, but MinerU 3.x uses the MinerU Open Source License—Apache-2.0 plus additional commercial conditions. That runtime is **not an approved ContextualWisdomLab commercial dependency**. Do not treat MinerU installation, a customer-supplied MinerU binary, or `Dockerfile.nvidia` as an approved commercial deployment path. Issue [#671](https://github.com/ContextualWisdomLab/newsdom-api/issues/671) owns replacement with an unrestricted commercial-use parser boundary. NewsDOM's own source remains MIT-licensed. -### Install +## What NewsDOM owns -Install `uv` first if it is not already available in your `PATH`, then sync the -repository-managed virtual environment: +- A stable `POST /parse` contract for PDF input and normalized document-tree output. +- Language and parse-mode request validation, including `mode=auto`, `ocr`, and `txt` compatibility semantics while the legacy adapter remains present. +- Fail-closed bearer authentication for `/parse` in the production profile. +- Separate unauthenticated `/health` liveness and `/ready` traffic-readiness semantics. +- Bounded, non-waiting parser admission with `NEWSDOM_MAX_CONCURRENT_PARSES` and an explicit `429 Too Many Requests` overload response. +- Request-scoped temporary workspace handling, sanitized parser failures, schema validation, and synthetic fixture provenance. -```bash -uv sync --frozen --all-extras -``` +NewsDOM does **not** make a third-party parser's license, model weights, OCR accuracy, or runtime availability part of its own MIT grant. It also does not turn `/health` into proof that parsing is ready. -To enable real parsing with MinerU, install the MinerU CLI separately in the -same `.venv` that `uv sync` created: +## Quick start + +### Install the repository-managed environment + +NewsDOM source metadata currently requires Python `>=3.10,<3.14`. ```bash -uv pip install --python .venv/bin/python "mineru[pipeline]==3.4.4" +uv sync --frozen --all-extras ``` -On Windows, replace `.venv/bin/python` with `.venv\Scripts\python.exe`. +`uv` creates the repository environment under `.venv`. On Windows the interpreter path is `.venv\Scripts\python.exe`; on macOS/Linux it is `.venv/bin/python`. -### Run +This command installs NewsDOM and its declared project dependencies only. It does **not** install the commercially restricted legacy MinerU runtime. + +### Run the API shell ```bash +export NEWSDOM_AUTH_MODE=required +export NEWSDOM_RUNTIME_PROFILE=production export NEWSDOM_API_TOKEN="$(openssl rand -hex 32)" +export NEWSDOM_MAX_CONCURRENT_PARSES=1 uv run uvicorn --app-dir src newsdom_api.main:app --reload ``` -### Docker +Check the two health levels independently: ```bash -docker build -t newsdom-api . -docker run -e NEWSDOM_AUTH_MODE=required \ - -e NEWSDOM_RUNTIME_PROFILE=production \ - -e NEWSDOM_API_TOKEN="$NEWSDOM_API_TOKEN" \ - -p 8000:8000 newsdom-api +curl -sS http://127.0.0.1:8000/health +curl -sS http://127.0.0.1:8000/ready ``` -The default image exposes the REST API on port `8000` as a multi-arch service -image. It is suitable for `linux/amd64` and `linux/arm64`, including Apple -Silicon hosts running the API service inside Docker. - -The default image ships the API service only and does not bundle the MinerU runtime. -`/parse` requires a compatible MinerU runtime to be available inside the container image or exposed through `NEWSDOM_MINERU_BIN`. - -> `GET /health` reports process liveness only. `GET /ready` reports traffic -> readiness and remains unavailable until both fail-closed authentication and -> the MinerU runtime are configured. Orchestrators must route traffic by -> `/ready`, not by `/health`. +`/health` answers for process liveness. `/ready` is the routing gate and must remain unavailable when the configured parser runtime or required authentication state cannot accept traffic. -#### docker compose +### Parse contract -A production-oriented `docker-compose.yml` is provided for standalone/sidecar -use. It requires `NEWSDOM_API_TOKEN` before startup and its healthcheck targets -`/ready`: +Once an **organization-approved parser backend** is configured, the public request shape remains: ```bash -docker compose up --build -``` - -Export `NEWSDOM_API_TOKEN` from a secret store before running Compose. Mount or -bundle MinerU and set `NEWSDOM_MINERU_BIN` when using the API-only image. The -service remains live but not ready until both dependencies are valid. - -#### Building a MinerU-bundled image - -To ship an image where `/parse` works behind a green `/health`, bundle a MinerU -runtime. Either build the NVIDIA variant below, or extend the default image and -install MinerU into the same virtualenv, for example: - -```dockerfile -FROM newsdom-api:latest -USER root -RUN uv pip install --python /app/.venv/bin/python "mineru[pipeline]==3.4.4" -USER newsdom +curl -F "file=@sample.pdf" \ + -H "Authorization: Bearer $NEWSDOM_API_TOKEN" \ + http://127.0.0.1:8000/parse ``` -Alternatively mount a MinerU executable and point `NEWSDOM_MINERU_BIN` at it, so -`/parse` no longer returns `503` behind a healthy `/health`. - -For heavier parsing deployments, build the optional NVIDIA-oriented variant: +Optional compatibility fields are `language` and `mode`: ```bash -docker build -f Dockerfile.nvidia -t newsdom-api:nvidia . -docker run --gpus all -p 8000:8000 newsdom-api:nvidia +curl -F "file=@sample.pdf" -F "language=ch" -F "mode=auto" \ + -H "Authorization: Bearer $NEWSDOM_API_TOKEN" \ + http://127.0.0.1:8000/parse ``` -`Dockerfile.nvidia` is intended for Linux/NVIDIA environments and is -`linux/amd64`-only. Apple Silicon can run the lean API image, but Docker -Desktop does not expose Apple GPU acceleration to Linux containers, so real -GPU-accelerated parsing should stay on a native Apple Silicon path instead of -the containerized runtime. - -The NVIDIA variant is `linux/amd64`-only and is meant for hosts that can -provide the CUDA user-space/runtime stack required by MinerU. - -### Parse a PDF +An explicit OCR-mode request uses the same authenticated boundary: ```bash -curl -F "file=@sample.pdf" \ +curl -F "file=@sample.pdf" -F "language=korean" -F "mode=ocr" \ -H "Authorization: Bearer $NEWSDOM_API_TOKEN" \ http://127.0.0.1:8000/parse ``` -`/parse` accepts `multipart/form-data` with a required `file` part -(`application/pdf`) and two optional form fields: +The current source's legacy adapter may return `503 Service Unavailable` when no parser runtime is available. That is preferable to silently treating an unapproved runtime as production-ready. -| Field | Default | Values | Maps to | -| ----- | ------- | ------ | ------- | -| `language` | `ch` | MinerU 3.4.4 public family or alias (`ch`, `en`, `japan`, `korean`, `arabic`, `east_slavic`, `cyrillic`, `devanagari`, …) | MinerU `-l` | -| `mode` | `auto` | `auto`, `ocr`, `txt` | MinerU `-m` | +## Capacity and failure behavior -`mode=auto` lets born-digital (text-layer) PDFs skip forced OCR; `ocr` forces -optical recognition and `txt` extracts only the embedded text layer. Invalid -values return `422`. The previous Japanese-newspaper behavior is still available -explicitly: +`NEWSDOM_MAX_CONCURRENT_PARSES` is an integer from `1` through `128` and defaults to `1` per process. Authentication runs before admission. When all parser leases are occupied, the service returns `429 Too Many Requests` with `Retry-After: 1` and `Cache-Control: no-store` before multipart body processing, temporary-file allocation, PDF validation, or parser execution. -```bash -curl -F "file=@sample.pdf" -F "language=japan" -F "mode=ocr" \ - -H "Authorization: Bearer $NEWSDOM_API_TOKEN" \ - http://127.0.0.1:8000/parse -``` +This is a last-resort per-process availability boundary, not a replacement for tenant-aware gateway limits or cluster resource quotas. Start with the conservative default, measure representative workloads, and scale replicas before raising per-process concurrency. -The accepted language contract follows the official -[MinerU 3.4.4 CLI implementation](https://github.com/opendatalab/MinerU/blob/mineru-3.4.4-released/mineru/utils/ocr_language.py). -MinerU canonicalizes `en`, `japan`, `chinese_cht`, and `latin` to `ch`; this -sidecar performs the same normalization before launching the subprocess. +Expected malformed-document/parser failures are translated into bounded client-facing responses; unexpected server faults remain observable through the server-error path rather than being mislabeled as caller input. -#### Authentication and readiness +## Containers -Parser authentication is required by default. This replaces the previous -default-open behavior. Configure the explicit production contract and supply the -secret from the deployment secret store: +The default container is the supported repository image surface: ```bash -export NEWSDOM_AUTH_MODE=required -export NEWSDOM_RUNTIME_PROFILE=production -export NEWSDOM_API_TOKEN=$(openssl rand -hex 32) -uv run uvicorn --app-dir src newsdom_api.main:app -curl -F "file=@sample.pdf" -H "Authorization: Bearer $NEWSDOM_API_TOKEN" \ - http://127.0.0.1:8000/parse +docker build -t newsdom-api . +docker run -p 8000:8000 \ + -e NEWSDOM_AUTH_MODE=required \ + -e NEWSDOM_RUNTIME_PROFILE=production \ + -e NEWSDOM_API_TOKEN="$NEWSDOM_API_TOKEN" \ + -e NEWSDOM_MAX_CONCURRENT_PARSES=1 \ + newsdom-api ``` -Missing or invalid caller credentials receive a fixed `401`; a missing required -server token returns a fixed `503` before the upload body is processed. `GET -/health` remains unauthenticated liveness. `GET /ready` succeeds only when the -authentication configuration and MinerU runtime can accept traffic. +The default image **ships the API service only** and **does not bundle the MinerU runtime**. It supports the repository's multi-architecture API shell, including Apple Silicon hosts running Linux containers, but `/ready` must stay fail-closed until an approved parser backend exists. -An isolated local development-only bypass is available only with the explicit -pair below: +`Dockerfile.nvidia` is retained in current source history and tests as the existing NVIDIA/MinerU distribution path, but it is **commercially blocked under organization policy** and must not be built, published, recommended, or treated as a supported release profile until #671 replaces the restricted parser stack. The same restriction applies even on an NVIDIA host; moving the component into another image or process does not cure its license terms. -```bash -NEWSDOM_AUTH_MODE=disabled \ -NEWSDOM_RUNTIME_PROFILE=development \ - uv run uvicorn --app-dir src newsdom_api.main:app --reload +## Architecture and integration boundary + +```text +client + -> FastAPI request/auth boundary + -> non-waiting parser admission + -> bounded temporary PDF workspace + -> parser adapter port + -> NewsDOM normalization + schema validation + -> canonical JSON response ``` -Do not use the development-only bypass as a production rollback. Restore a -working secret or the previous release instead. +The parser adapter is an implementation dependency. The stable product responsibility is the request, safety, evidence, and normalized-output contract around it. The replacement work in #671 should preserve that boundary rather than make downstream callers depend on another parser's internal types. + +For deeper design and current-vs-target maturity, see [ARCHITECTURE.md](ARCHITECTURE.md), the [ADR index](docs/adr/README.md), and the canonical documentation under [`docs/`](docs/). -Each request is written to a request-scoped temporary directory before MinerU -runs, and those temporary files are removed after the response completes. -Sanitized parse failures return `503 MinerU runtime unavailable` when the -runtime cannot be executed and `502 MinerU output was incomplete` when MinerU -finishes without the required output artifacts. +## Verification -### Run tests +Run the repository suite: ```bash uv run pytest ``` -### Fuzzing smoke +The repository quality gate requires 100% configured source coverage and docstring audit coverage. Security, dependency, image, fuzzing, and package checks run independently in CI. + +A local fuzzing smoke remains available against committed synthetic fixtures: ```bash uv run python fuzzers/dom_builder_fuzzer.py --smoke tests/fixtures/mineru_sample.json @@ -199,42 +143,29 @@ uv run python fuzzers/schema_response_fuzzer.py --smoke fuzzers/corpus/schema_re uv run python fuzzers/equivalence_metrics_fuzzer.py --smoke fuzzers/corpus/equivalence_metrics_fuzzer/structural_metrics.json ``` -Every `fuzzers/*_fuzzer.py` target is coverage-guided under Atheris and is -picked up automatically by the ClusterFuzzLite workflow, which runs a bounded -budget on each pull request. Targets cover the untrusted-input boundaries: the -MinerU DOM normalizer (`build_dom`), the `ParseResponse` schema validator, and -the equivalence metrics normalizer. See `docs/papers/` for background. - -The repository also enforces a `quality-gate` workflow with 100% source -coverage and docstring audit coverage. - -## Fixtures and provenance +The historical `mineru_sample.json` fixture name records parser provenance; it is not an instruction to install the MinerU runtime. -This repository ships only synthetic test fixtures and derived structural -baselines. For fixture provenance and regeneration notes, see -`tests/fixtures/README.md`. +## Documentation -## Development +- [Public manual](manual/index.md) — product and integration entry point. +- [Architecture](ARCHITECTURE.md) — system ownership and current/target boundaries. +- [Security](SECURITY.md) — vulnerability reporting and security posture. +- [Contributing](CONTRIBUTING.md) — development and validation workflow. +- [Fixtures and provenance](tests/fixtures/README.md) — synthetic fixture source and regeneration rules. +- [Git flow](docs/workflow/git-flow.md) — repository branch workflow. +- [Changelog](CHANGELOG.md) — source change history; not release evidence by itself. +- [Commercial parser replacement #671](https://github.com/ContextualWisdomLab/newsdom-api/issues/671) — current parser-license blocker and acceptance contract. -Development setup, fixture handling rules, and local-only baseline -maintenance are documented in `CONTRIBUTING.md`. +GitHub Pages source lives under `manual/`. A source commit is not proof that the public site deployed; publication requires the Pages workflow to finish and the live HTTPS content to be re-read. -Mechanical branch updates and merges are attributed to `github-actions[bot]`. -Scratch PoC files are not committed. Failed GitHub Checks are not reviewed as URL lists. -OpenCode Review, Strix Security Scan, and PR Review Merge Scheduler are -provided by the organization-level required workflows in -`ContextualWisdomLab/.github`, not copied into this repository. +## Status -Security reporting guidance is documented in `SECURITY.md`. -Version tags trigger a GitHub-native release workflow that builds -distribution artifacts, checksums, and provenance attestations. +`pyproject.toml` currently records NewsDOM source version `0.2.0`. Source metadata, an open PR, or a passing check is not by itself a published-release or commercial-readiness claim. Consult [GitHub Releases](https://github.com/ContextualWisdomLab/newsdom-api/releases) for immutable published release evidence. -Project history is tracked in `CHANGELOG.md`. +The current parser-license blocker means the repository must not be represented as commercially complete even when NewsDOM-owned tests are green. -Repository branch workflow is documented in `docs/workflow/git-flow.md`. +## License -## Repository layout +NewsDOM API original source and documentation are licensed under the [MIT License](LICENSE). Third-party packages, parser runtimes, model weights, container bases, fonts, datasets, and other external assets retain their own licenses and distribution terms. -- `src/newsdom_api/`: API, MinerU wrapper, DOM builder, synthetic fixture generator -- `tests/`: unit tests and committed synthetic fixtures -- `tools/`: local maintenance utilities +The MIT grant does not relicense the legacy MinerU runtime. Under current ContextualWisdomLab policy, that runtime is outside the approved commercial inbound baseline and remains blocked by #671. diff --git a/SECURITY.md b/SECURITY.md index e80f8ecf..c63654e9 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,13 +2,17 @@ ## Reporting a vulnerability -Please report a vulnerability privately by opening a GitHub Security -Advisory draft for this repository: +Please report a vulnerability privately through this repository's GitHub +Security Advisory workflow when it is available: -- [GitHub Security Advisory draft](https://github.com/Seongho-Bae/newsdom-api/security/advisories/new) -- [Maintainer profile](https://github.com/seonghobae) +- [GitHub Security Advisory draft](https://github.com/ContextualWisdomLab/newsdom-api/security/advisories/new) -Do not open a public issue for an unpatched vulnerability. +If that private repository feature is unavailable to you, contact the +ContextualWisdomLab repository maintainers through an established private +channel rather than disclosing the issue publicly. + +Do not open a public issue, pull-request comment, or discussion for an +unpatched vulnerability. When reporting, include: diff --git a/docker-compose.yml b/docker-compose.yml index 19bdffa3..a290571f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,6 +15,9 @@ services: NEWSDOM_AUTH_MODE: required NEWSDOM_RUNTIME_PROFILE: production NEWSDOM_API_TOKEN: "${NEWSDOM_API_TOKEN:?set NEWSDOM_API_TOKEN}" + # One process-local MinerU lease is the conservative production default. + # Raise this only after measuring peak RSS/VRAM and parser latency. + NEWSDOM_MAX_CONCURRENT_PARSES: "1" # Point this at a bundled or mounted executable when the default API-only # image is used. The NVIDIA image already supplies its MinerU runtime. # NEWSDOM_MINERU_BIN: /opt/mineru/bin/mineru diff --git a/docs/benchmarks/upload-ingestion-result.schema.json b/docs/benchmarks/upload-ingestion-result.schema.json new file mode 100644 index 00000000..3ec93136 --- /dev/null +++ b/docs/benchmarks/upload-ingestion-result.schema.json @@ -0,0 +1,342 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/ContextualWisdomLab/newsdom-api/blob/develop/docs/benchmarks/upload-ingestion-result.schema.json", + "title": "NewsDOM upload ingestion benchmark result", + "description": "Raw, reproducible evidence for issue #534 chunk-size and concurrency candidates.", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "schema_path", + "generated_at", + "benchmark_environment", + "fixtures", + "cases" + ], + "properties": { + "schema_version": { + "const": "1.2.0" + }, + "schema_path": { + "const": "docs/benchmarks/upload-ingestion-result.schema.json" + }, + "generated_at": { + "type": "string", + "format": "date-time" + }, + "benchmark_environment": { + "$ref": "#/$defs/benchmark_environment" + }, + "fixtures": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/fixture_record" + } + }, + "cases": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/benchmark_case" + } + } + }, + "$defs": { + "benchmark_environment": { + "type": "object", + "additionalProperties": false, + "required": [ + "python_version", + "python_implementation", + "executable", + "platform", + "machine", + "processor_count", + "rss_provider", + "execution_image", + "cpu_model", + "memory_bytes", + "storage_device", + "filesystem", + "dependency_lock_sha256", + "commit_sha", + "worker_count", + "cache_state" + ], + "properties": { + "python_version": { + "type": "string", + "minLength": 1 + }, + "python_implementation": { + "type": "string", + "minLength": 1 + }, + "executable": { + "type": "string", + "minLength": 1 + }, + "platform": { + "type": "string", + "minLength": 1 + }, + "machine": { + "type": "string" + }, + "processor_count": { + "type": [ + "integer", + "null" + ], + "minimum": 1 + }, + "rss_provider": { + "const": "linux_proc_status_vmrss" + }, + "execution_image": { + "type": "string", + "minLength": 1 + }, + "cpu_model": { + "type": "string", + "minLength": 1 + }, + "memory_bytes": { + "type": "integer", + "minimum": 1 + }, + "storage_device": { + "type": "string", + "minLength": 1 + }, + "filesystem": { + "type": "string", + "minLength": 1 + }, + "dependency_lock_sha256": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "commit_sha": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, + "worker_count": { + "type": "integer", + "minimum": 1 + }, + "cache_state": { + "enum": [ + "cold", + "warm", + "disabled" + ] + } + } + }, + "fixture_record": { + "type": "object", + "additionalProperties": false, + "required": [ + "fixture_name", + "size_bytes", + "sha256" + ], + "properties": { + "fixture_name": { + "type": "string", + "minLength": 1 + }, + "size_bytes": { + "type": "integer", + "minimum": 1, + "maximum": 20971520 + }, + "sha256": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + } + } + }, + "benchmark_case": { + "type": "object", + "additionalProperties": false, + "required": [ + "fixture_name", + "candidate", + "chunk_bytes", + "concurrency", + "repetitions", + "metrics", + "cohort_observations", + "samples" + ], + "properties": { + "fixture_name": { + "type": "string", + "minLength": 1 + }, + "candidate": { + "enum": [ + "8kib", + "64kib", + "256kib", + "1mib", + "adaptive" + ] + }, + "chunk_bytes": { + "type": "integer", + "minimum": 8192, + "maximum": 1048576 + }, + "concurrency": { + "type": "integer", + "minimum": 1 + }, + "repetitions": { + "type": "integer", + "minimum": 1 + }, + "metrics": { + "type": "object", + "additionalProperties": false, + "required": [ + "p50_latency_seconds", + "p95_latency_seconds", + "throughput_bytes_per_second", + "read_calls", + "cpu_seconds", + "peak_rss_bytes", + "peak_rss_delta_bytes", + "temporary_disk_bytes", + "event_loop_p95_delay_seconds", + "event_loop_max_delay_seconds" + ], + "properties": { + "p50_latency_seconds": { + "type": "number", + "minimum": 0 + }, + "p95_latency_seconds": { + "type": "number", + "minimum": 0 + }, + "throughput_bytes_per_second": { + "type": "number", + "minimum": 0 + }, + "read_calls": { + "type": "integer", + "minimum": 1 + }, + "cpu_seconds": { + "type": "number", + "minimum": 0 + }, + "peak_rss_bytes": { + "type": "integer", + "minimum": 0 + }, + "peak_rss_delta_bytes": { + "type": "integer", + "minimum": 0 + }, + "temporary_disk_bytes": { + "type": "integer", + "minimum": 1 + }, + "event_loop_p95_delay_seconds": { + "type": "number", + "minimum": 0 + }, + "event_loop_max_delay_seconds": { + "type": "number", + "minimum": 0 + } + } + }, + "cohort_observations": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/cohort_observation" + } + }, + "samples": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/sample_record" + } + } + } + }, + "cohort_observation": { + "type": "object", + "additionalProperties": false, + "required": [ + "wall_seconds", + "cpu_seconds", + "rss_baseline_bytes", + "peak_rss_bytes", + "peak_rss_delta_bytes", + "event_loop_delay_samples_seconds" + ], + "properties": { + "wall_seconds": { + "type": "number", + "minimum": 0 + }, + "cpu_seconds": { + "type": "number", + "minimum": 0 + }, + "rss_baseline_bytes": { + "type": "integer", + "minimum": 0 + }, + "peak_rss_bytes": { + "type": "integer", + "minimum": 0 + }, + "peak_rss_delta_bytes": { + "type": "integer", + "minimum": 0 + }, + "event_loop_delay_samples_seconds": { + "type": "array", + "items": { + "type": "number", + "minimum": 0 + } + } + } + }, + "sample_record": { + "type": "object", + "additionalProperties": false, + "required": [ + "duration_seconds", + "read_calls", + "bytes_copied" + ], + "properties": { + "duration_seconds": { + "type": "number", + "minimum": 0 + }, + "read_calls": { + "type": "integer", + "minimum": 1 + }, + "bytes_copied": { + "type": "integer", + "minimum": 1, + "maximum": 20971520 + } + } + } + } +} diff --git a/docs/doctoring/bounded-parser-admission.md b/docs/doctoring/bounded-parser-admission.md new file mode 100644 index 00000000..55a1e759 --- /dev/null +++ b/docs/doctoring/bounded-parser-admission.md @@ -0,0 +1,226 @@ +# Bounded parser admission + +## Decision + +NewsDOM applies a process-local, non-waiting concurrency boundary to the +expensive synchronous `/parse` path. The immutable setting +`NEWSDOM_MAX_CONCURRENT_PARSES` accepts an integer from `1` through `128` and +defaults to `1`. + +An authenticated request must acquire one parser lease before FastAPI or +Starlette parses the multipart body, before request bytes are copied to a +temporary file, before PDF structure validation, and before MinerU is invoked. +If no lease is available, the service immediately returns: + +```http +HTTP/1.1 429 Too Many Requests +Retry-After: 1 +Cache-Control: no-store, no-cache, max-age=0 +``` + +The service does not wait for a lease and does not create an in-process upload +queue. + +## Buyer-visible problem + +MinerU parsing can consume substantial CPU, memory, temporary storage, and GPU +resources. Without an admission boundary, a short request burst can begin many +multipart reads and parser processes at once. The resulting resource +competition can increase latency, trigger out-of-memory termination, exhaust +writable volumes, and make every request fail rather than rejecting only excess +work. + +OWASP API4:2023 identifies unrestricted consumption of CPU, memory, process, +payload, and third-party resources as an API availability and cost risk. The +control implemented here limits active parser work rather than relying on +client goodwill or a best-effort timeout after resources have already been +allocated. + +## Boundary ordering + +The access path is deliberately ordered as follows: + +```mermaid +flowchart LR + A[POST /parse] --> B[Raw bearer authentication] + B -->|invalid| C[401 or configuration 503] + B -->|valid| D[Non-waiting parser lease] + D -->|saturated| E[429 + Retry-After: 1 + no-store] + D -->|admitted| F[Multipart body] + F --> G[Upload and PDF validation] + G --> H[MinerU] + H --> I[DOM response] + I --> J[Lease release] + H -->|failure or cancellation| J +``` + +Authentication precedes admission so unauthenticated traffic cannot consume a +parser lease or infer capacity from overload responses. Admission precedes +multipart parsing so rejected work does not retain request bodies, allocate +request-scoped temporary files, validate PDFs, or enter MinerU. + +## HTTP semantics + +RFC 6585 defines `429 Too Many Requests` for request-rate or overload controls +and permits a `Retry-After` response field. NewsDOM uses an exact one-second +hint because the synchronous API has no durable queue position or completion +estimate. Clients should still use bounded jitter and stop retrying according +to their own deadline. + +RFC 6585 also states that 429 responses must not be stored by a cache. RFC 9111 +provides the `no-store` response directive. NewsDOM applies `Cache-Control: +no-store, no-cache, max-age=0` through the same security-header middleware used +for every parser response, preventing an intermediary from replaying a stale +overload decision after capacity has recovered. + +The response body is fixed and does not reveal the configured capacity, current +lease count, parser identity, queue depth, file metadata, internal paths, or +another tenant's activity. + +## Implementation + +`src/newsdom_api/admission.py` contains `ParseAdmissionLimiter`. Each FastAPI +application owns one limiter. The implementation uses +`threading.BoundedSemaphore` because the protected work executes through a +thread-backed synchronous parser boundary and requests may arrive from multiple +threads or event-loop tasks. + +`BoundedSemaphore` also treats release beyond the configured initial value as a +programming error. This is preferable to silently increasing capacity after a +double release. + +The middleware follows this invariant: + +```text +authenticate +→ try_acquire without waiting +→ call downstream application +→ release in finally +``` + +The `finally` boundary returns the lease after a successful response, form or +PDF validation failure, MinerU runtime failure, incomplete MinerU output, +unhandled exception, or request cancellation. Authentication and service +configuration failures occur before acquisition and therefore do not require a +release. + +## Scope and MSA behavior + +The capacity is **per process**. It is not an organization-wide, cluster-wide, +repository-wide, or tenant-wide counter. If a replica has `P` serving processes +and each process is configured with `N`, that replica can nominally run at most +`P × N` parser jobs. With `R` identical replicas, the nominal deployment ceiling +is `R × P × N`. + +Actual safe capacity can be lower because pods share CPU, memory, storage, +model caches, and sometimes a GPU. A gateway may add identity-aware request +rates, quotas, fairness, and circuit breaking. A future durable job service may +own persistent queueing and cancellation. Neither layer should remove this +last-resort process boundary. + +The module remains usable independently in the standalone NewsDOM service and +when NewsDOM is imported as a sidecar or MSA component. It does not require a +shared database, distributed lock, Redis, or naruon runtime. + +## Configuration and deployment + +The conservative production default is: + +```text +NEWSDOM_MAX_CONCURRENT_PARSES=1 +``` + +The setting is parsed once during application creation and remains immutable. +Changing the environment of a running process does not resize the active +limiter. Operators must restart the serving process after an approved change. + +The repository-owned Docker Compose and Kubernetes examples explicitly set the +value to `1`. The Kubernetes example has two replicas, so it advertises two +nominal active parses while retaining independent failure domains. + +Raise the value only after measuring representative PDFs across the expected +size, page-count, language, OCR mode, and document-complexity distribution. +Record at least: + +- peak resident memory and, when applicable, peak VRAM; +- temporary-volume peak usage; +- p50, p95, and p99 parser latency; +- timeout, 429, 502, 503, and process-crash rates; +- output accuracy and structural-equivalence metrics; +- behavior when one replica is terminating or unavailable. + +Prefer additional replicas when one MinerU process already consumes a large +share of a pod's memory or GPU budget. + +## Verification evidence + +The regression suite proves: + +- immutable direct and environment configuration with a default of one; +- rejection of blank, non-integer, boolean, zero, and values above 128; +- application-local rather than global lease state; +- immediate saturation without a body read or downstream call; +- fixed 429 body, exact `Retry-After: 1`, and no-store response headers; +- lease recovery after success, exception, and cancellation; +- authentication failure without lease consumption; +- a realistic burst of 32 requests with capacity four, admitting four and + rejecting the remaining 28; +- OpenAPI publication of the 429 response; +- production deployment examples and operator documentation; +- 100% production statement and branch coverage and public docstrings. + +A live deployment acceptance test should additionally hold all configured +leases at the MinerU boundary, send one excess authenticated request, verify no +new temporary file or parser process is created, release one request, and prove +the next request is admitted. + +## Failure handling + +Unexpected 429 growth means the service is protecting itself. Operators should +inspect traffic composition, parser duration, replica health, CPU, RSS, VRAM, +temporary storage, and downstream dependencies before raising the limit. +Readiness should remain a dependency and security signal; ordinary short-lived +saturation does not make the process permanently unready. + +A leaked lease would reduce capacity until process restart. The test suite +covers every current response and cancellation path, while +`BoundedSemaphore` detects over-release. Monitoring should alert on sustained +429 rate, falling successful throughput, parser duration, and replica restart +rate rather than exposing internal lease counts to unauthenticated callers. + +## Rollback + +Rollback must preserve authentication, upload limits, and non-waiting +admission. Reduce `NEWSDOM_MAX_CONCURRENT_PARSES` to the last accepted value and +restart processes so the immutable setting is reloaded. If a release regression +is suspected, restore the previous verified image digest together with its +recorded capacity. + +Before reopening traffic, verify `/health`, `/ready`, one successful +authenticated parse, one intentional saturated 429, and successful admission +after the held request releases its lease. Do not roll back to an unbounded or +default-open parser. + +## Limitations and next slices + +This boundary does not provide durable asynchronous processing, tenant +fairness, distributed capacity, priority scheduling, or a job status and +cancellation API. Those capabilities belong in a separately bounded job +service. The synchronous endpoint intentionally exposes overload instead of +pretending to queue work it cannot persist or recover. + +## References + +Fielding, R., Nottingham, M., & Reschke, J. (2022). *HTTP caching* (RFC +9111). Internet Engineering Task Force. https://doi.org/10.17487/RFC9111 + +Nottingham, M., & Fielding, R. (2012). *Additional HTTP status codes* (RFC +6585). Internet Engineering Task Force. https://doi.org/10.17487/RFC6585 + +OWASP Foundation. (2023). *API4:2023 unrestricted resource consumption*. +*OWASP API Security Top 10*. +https://owasp.org/API-Security/editions/2023/en/0xa4-unrestricted-resource-consumption/ + +Python Software Foundation. (2026). *threading—Thread-based parallelism +(Python 3.10.20 documentation)*. +https://docs.python.org/3.10/library/threading.html diff --git a/docs/doctoring/upload-ingestion-benchmark-methodology.md b/docs/doctoring/upload-ingestion-benchmark-methodology.md new file mode 100644 index 00000000..283007a2 --- /dev/null +++ b/docs/doctoring/upload-ingestion-benchmark-methodology.md @@ -0,0 +1,181 @@ +# Upload-ingestion benchmark methodology + +## Purpose + +This record defines the reproducible evidence required before NewsDOM changes +the current 8 KiB upload read size. It deliberately does **not** select a new +default or claim a performance improvement. The benchmark compares every +candidate required by issue #534 while the production parser admission boundary +remains fixed and independently testable. + +FastAPI documents that the asynchronous `UploadFile` methods run the underlying +file operations in a thread pool. Starlette documents that `UploadFile` shares +AnyIO's thread-pool capacity with synchronous endpoints, file responses, +background tasks, and other framework work. Chunk size therefore affects more +than sequential file-copy throughput: it changes the number of thread-pool +crossings, event-loop responsiveness, and the amount of data retained by each +concurrent request. + +## Candidate matrix + +The default command measures the following exact candidates: + +| Candidate | Exact or resolved read size | +| --- | ---: | +| `8kib` | 8,192 bytes | +| `64kib` | 65,536 bytes | +| `256kib` | 262,144 bytes | +| `1mib` | 1,048,576 bytes | +| `adaptive` | 64 KiB at 1 MiB, 256 KiB through 5 MiB, otherwise 1 MiB | + +Every candidate is measured at concurrency `1`, `8`, `32`, and `128`. The +fixture set must contain representative PDFs near 1 MiB, 5 MiB, and the existing +20 MiB request limit. Fixtures remain outside public git when licensing, +privacy, or contractual restrictions apply. The raw report stores each fixture +name, byte size, and SHA-256 identifier so an internal evidence bundle can be +audited without publishing document content. + +The adaptive rule is only a benchmark candidate. It is not production behavior +and must not be described as selected until a reviewed result satisfies the +acceptance rules below. + +## Reproducible command + +```bash +uv run python tools/benchmark_upload_ingestion.py \ + --fixtures-dir /absolute/path/to/approved-pdf-fixtures \ + --environment-manifest /absolute/path/to/environment.json \ + --output artifacts/upload-ingestion-result.json \ + --repetitions 3 +``` + +The environment manifest is a strict JSON object containing the execution image, +CPU model, memory bytes, storage device, filesystem, dependency-lock SHA-256, +exact 40-character commit SHA, worker count, and cache state +(`cold`, `warm`, or `disabled`). The harness rejects missing and unknown +fields. The default command runs the complete candidate and concurrency matrix. Filters +are available for diagnosis, but a filtered run is not sufficient for a default +selection. Execute the benchmark on an otherwise idle, dedicated Linux +environment and record the container or host image, CPU model, memory, storage +class, filesystem, Python and dependency lock, commit SHA, worker count, and +whether the files were cold or warm in the operating-system cache. + +## Measurement implementation + +`tools/benchmark_upload_ingestion.py` uses a real file-backed Starlette +`UploadFile` and copies each request to an independent temporary file. It does +not use an in-memory bytes object as the source of accepted evidence. + +The harness retains both request-level and cohort-level raw observations and +derives: + +- p50 and p95 request latency by deterministic nearest-rank percentile; +- aggregate copied bytes divided by cohort wall time; +- total `UploadFile.read(size)` calls; +- process CPU time; +- each cohort's current-RSS baseline, sampled peak RSS, and peak-minus-baseline + delta; +- temporary bytes written; +- raw event-loop scheduling-delay samples, their nearest-rank p95, and maximum. + +Python defines `time.perf_counter()` as a high-resolution monotonic clock suited +to duration measurement and `time.process_time()` as process CPU time excluding +sleep. Event-loop responsiveness is reported as a real nearest-rank p95 over the +retained probe samples, not as a maximum mislabeled as a percentile. + +On Linux, the harness samples `VmRSS` from `/proc/self/status` during each +cohort. The Linux kernel distinguishes `VmRSS`, the current resident set, from +`VmHWM`, the process-lifetime resident-set high-water mark. A lifetime high-water +metric would contaminate later matrix cases with memory used by earlier cases; +therefore `resource.getrusage(...).ru_maxrss` is not accepted as case-local +memory evidence. Every report preserves the cohort baseline and sampled peak so +the absolute process budget and incremental case pressure can both be audited. + +If `/proc/self/status` is unavailable or malformed, the provider returns zero +and the environment record still identifies `linux_proc_status_vmrss`. Such a +run does not satisfy the final memory-evidence gate. Other operating systems +require a separately reviewed current-RSS provider before their memory numbers +can support a production default. + +RSS sampling is observational rather than an allocation profiler. Candidate +selection therefore requires repeated clean-process runs and should be +corroborated with operating-system or container memory telemetry. Do not treat +`peak_rss_delta_bytes` as exact ownership of memory that may be shared, +allocator-retained, or concurrently used by another thread. + +## Raw evidence schema + +The checked-in JSON Schema is: + +```text +docs/benchmarks/upload-ingestion-result.schema.json +``` + +Accepted evidence must validate against schema version `1.2.0`. The report keeps +request samples and raw cohort observations alongside aggregates, rejects +unknown top-level and case fields, and records the exact resolved byte size even +for the adaptive candidate. Generated reports are evidence artifacts, not source +files; do not commit private fixture paths or customer document names. + +## Interpretation and selection gates + +A candidate can replace 8 KiB only when the exact current-head report shows all +of the following across the complete matrix: + +1. no upload-size, magic-byte, temporary-file cleanup, authentication, or parser + admission regression; +2. no materially worse request p95 latency or event-loop p95 at high + concurrency, with the maximum retained for tail diagnosis; +3. absolute sampled peak RSS and peak-minus-baseline pressure remain inside an + explicit per-process budget; +4. thread-pool read-call reduction does not merely shift cost to larger retained + buffers or storage stalls; +5. results are reproduced on at least two runs from clean processes; +6. the selected rule is simple enough to explain, test, operate, and roll back; +7. the production regression test, README, architecture, runbook, CHANGELOG, and + release evidence all identify the same value or rule. + +Compare distributions and operational thresholds rather than publishing a +single mean. A percentage improvement may be stated only when the attached raw +report, environment manifest, analysis code, and variability treatment support +it. + +## Accuracy and security invariants + +Chunk experiments must preserve: + +- authentication and process admission before multipart body consumption; +- first-five-byte PDF magic rejection; +- the exact 20 MiB streamed limit even when size metadata is missing; +- no write of the chunk that crosses the limit; +- request-scoped temporary files and cleanup after every outcome; +- structural PDF validation before MinerU; +- parser-not-called evidence for rejected requests; +- fixed sanitized errors and no-store response headers. + +The benchmark is not permission to bypass `NEWSDOM_MAX_CONCURRENT_PARSES`. +Concurrency cases measure ingestion behavior under controlled experiments; a +production deployment must continue to reject active parser work above its +accepted process budget. + +## Rollback + +Keep 8 KiB as the rollback value until a new policy has exact-head benchmark and +release acceptance. If a deployed policy increases RSS, event-loop delay, +temporary-storage pressure, timeout rate, or parser failures, restore the prior +value or release, restart processes so immutable configuration and code are +loaded, and rerun the overload and upload-limit smoke tests. + +## References + +FastAPI. (2026). *Request files*. FastAPI documentation. +https://fastapi.tiangolo.com/tutorial/request-files/ + +Linux Kernel Organization. (2026). *The /proc filesystem*. Linux kernel +documentation. https://www.kernel.org/doc/html/latest/filesystems/proc.html + +Python Software Foundation. (2026). *time—Time access and conversions (Python +3.14.6 documentation)*. https://docs.python.org/3/library/time.html + +Starlette. (2026). *Thread pool*. Starlette documentation. +https://www.starlette.io/threadpool/ diff --git a/docs/operations/deploy-runbook.md b/docs/operations/deploy-runbook.md index 760d0987..b01de439 100644 --- a/docs/operations/deploy-runbook.md +++ b/docs/operations/deploy-runbook.md @@ -16,6 +16,7 @@ release acceptance record. NEWSDOM_AUTH_MODE=required NEWSDOM_RUNTIME_PROFILE=production NEWSDOM_API_TOKEN= + NEWSDOM_MAX_CONCURRENT_PARSES=1 ``` 3. `uv run uvicorn --app-dir src newsdom_api.main:app --host 127.0.0.1 --port 8000` @@ -31,13 +32,14 @@ configuration and the MinerU executable are ready to accept traffic. ## Container smoke 1. `docker build -t newsdom-api:unreleased .` -2. Run with an explicit secret and production settings: +2. Run with an explicit secret, production settings, and process budget: ```bash docker run --rm -p 18080:8000 \ -e NEWSDOM_AUTH_MODE=required \ -e NEWSDOM_RUNTIME_PROFILE=production \ -e NEWSDOM_API_TOKEN="$NEWSDOM_API_TOKEN" \ + -e NEWSDOM_MAX_CONCURRENT_PARSES=1 \ newsdom-api:unreleased ``` @@ -53,6 +55,46 @@ runtime, so container smoke should treat `/parse` as an external-runtime check rather than a default-image contract. A container may be live while remaining unready; never route production traffic from `/health` alone. +## Parser admission and overload smoke + +`NEWSDOM_MAX_CONCURRENT_PARSES` accepts an integer from `1` through `128` and +defaults to `1`. The budget is **per process**. It is snapshotted when the +application is created and cannot be changed by mutating the environment later. +A deployment with `R` one-process replicas and a value of `N` has a nominal +active-parser ceiling of `R × N`. + +Authentication is evaluated before capacity. After successful authentication, +the service attempts one non-waiting parser lease **before the multipart body**, +temporary-file allocation, PDF validation, or MinerU. When all leases are in +use, the request receives: + +```http +HTTP/1.1 429 Too Many Requests +Retry-After: 1 +Cache-Control: no-store, no-cache, max-age=0 +``` + +No in-process waiting queue is created. The lease is released after success, +validation failure, MinerU failure, request cancellation, or another exception. +A gateway may add tenant-aware request-rate policy, but it must not convert the +synchronous endpoint into an unbounded buffered queue. + +To exercise the contract before rollout: + +1. Use a synthetic or non-sensitive representative PDF and configure a test + instance with `NEWSDOM_MAX_CONCURRENT_PARSES=1`. +2. Hold one authenticated parse active at the MinerU boundary. +3. Send a second authenticated parse and verify the fixed 429, exact + `Retry-After: 1`, and no-store cache policy. +4. Confirm no second temporary upload or MinerU process appears. +5. Release the first request, then verify the next parse is admitted. This proves + that success and failure paths do not leak leases. + +Start at one lease. Increase only after measuring peak RSS, peak VRAM when +applicable, temporary storage, p95/p99 parser latency, timeout rate, and output +accuracy on representative 1/5/20 MiB PDFs. Prefer additional replicas when one +parser already consumes a substantial share of the pod memory or GPU budget. + ## Kubernetes smoke 1. Replace `ghcr.io/contextualwisdomlab/newsdom-api:unreleased` in @@ -64,13 +106,17 @@ unready; never route production traffic from `/health` alone. 3. Ensure the MinerU executable and models are present in the deployed image or supplied through a separately reviewed read-only volume. The example exposes bounded writable `emptyDir` volumes only for `/tmp` and the runtime cache. -4. Apply the manifest and verify that the namespace enforces the Restricted Pod +4. Confirm `NEWSDOM_MAX_CONCURRENT_PARSES` is explicitly set per process. The + reference has two replicas at one lease each, for two nominal active parses. +5. Apply the manifest and verify that the namespace enforces the Restricted Pod Security profile. -5. Confirm the pod runs as UID/GID 10001 with `RuntimeDefault` seccomp, +6. Confirm the pod runs as UID/GID 10001 with `RuntimeDefault` seccomp, privilege escalation disabled, all Linux capabilities dropped, and a read-only root filesystem. -6. Verify `GET /health` for liveness and `GET /ready` for Service endpoint +7. Verify `GET /health` for liveness and `GET /ready` for Service endpoint readiness before sending authenticated parser traffic. +8. Run the overload smoke through the Service and verify excess work is spread + across replicas or receives the fixed 429 rather than accumulating in pods. The repository-owned GHCR path is the intended image source. Production cluster admission policy must allow only the reviewed organization/repository path and @@ -86,6 +132,8 @@ a cluster-specific registry trust policy. - After a tag push or manual dispatch, verify the GitHub Release contains `SHA256SUMS.txt`, `release-manifest.json`, and `*.intoto.jsonl` assets. - Record the accepted image digest and use that immutable digest in Kubernetes. +- Record the accepted parser capacity, replica count, load profile, peak resource + evidence, and rollback threshold in the deployment acceptance record. ## Failure handling @@ -94,19 +142,33 @@ a cluster-specific registry trust policy. - Expect `/parse` failures to stay sanitized: use generic client-facing details for `503 Service Unavailable` when the backend runtime cannot execute and `502 Bad Gateway` when required OCR artifacts are missing or invalid. +- Treat unexpected 429 growth as capacity pressure, not an authentication or + readiness failure. Inspect traffic, input-size mix, parser latency, RSS/VRAM, + replica availability, and downstream storage before changing the limit. - Reconcile the failure against `README.md`, `CHANGELOG.md`, package/OpenAPI versions, and the relevant workflow before closing the task. +## Rollback + +A safe admission rollback changes configuration or restores the previous release; +it never removes authentication, upload limits, or the non-waiting boundary. +Reduce `NEWSDOM_MAX_CONCURRENT_PARSES` to the last accepted value and restart the +processes so immutable settings are reloaded. If the new release itself is at +fault, restore the prior digest and its recorded capacity. Verify health, +readiness, one successful authenticated parse, one saturated 429, and subsequent +lease recovery before reopening traffic. + ## Fail-closed authentication migration for planned 0.3.0 Parser authentication is required by default. The previous default-open behavior -must not be carried into a production or shared deployment. Configure all three +must not be carried into a production or shared deployment. Configure all four settings before routing traffic: ```text NEWSDOM_AUTH_MODE=required NEWSDOM_RUNTIME_PROFILE=production NEWSDOM_API_TOKEN= +NEWSDOM_MAX_CONCURRENT_PARSES=1 ``` Use `GET /health` for process liveness and `GET /ready` for traffic readiness. diff --git a/docs/operations/kubernetes-deployment.yaml b/docs/operations/kubernetes-deployment.yaml index f11f1f2c..79485ece 100644 --- a/docs/operations/kubernetes-deployment.yaml +++ b/docs/operations/kubernetes-deployment.yaml @@ -64,6 +64,10 @@ spec: secretKeyRef: name: newsdom-api-runtime key: api_token + - name: NEWSDOM_MAX_CONCURRENT_PARSES + # Capacity is per process and per replica; this two-replica + # example therefore admits at most two active parses at once. + value: "1" - name: NEWSDOM_MINERU_BIN value: /opt/mineru/bin/mineru volumeMounts: diff --git a/docs/security/api-security-checklist.md b/docs/security/api-security-checklist.md index 86be69fd..b670327e 100644 --- a/docs/security/api-security-checklist.md +++ b/docs/security/api-security-checklist.md @@ -5,6 +5,7 @@ Apply this checklist whenever the FastAPI surface changes. ## In-scope endpoints - `/health` +- `/ready` - `/parse` - `/docs` - `/redoc` @@ -12,18 +13,43 @@ Apply this checklist whenever the FastAPI surface changes. ## Baseline checks - validate upload handling and content-type expectations for `/parse` -- ensure error messages do not leak private reference paths, - secrets, or credentials -- keep synthetic fixtures in tests and examples; never use private - reference inputs in public evidence -- verify request handling fails safely when MinerU is missing or - returns incomplete outputs -- confirm docs endpoints remain informational and do not imply - unsupported authentication or execution guarantees +- ensure error messages do not leak private reference paths, secrets, + credentials, configured capacity, active lease counts, or another tenant's + activity +- keep synthetic fixtures in tests and examples; never use private reference + inputs in public evidence +- verify request handling fails safely when MinerU is missing or returns + incomplete outputs +- confirm `/health` is liveness-only and `/ready` represents authentication + configuration plus MinerU runtime availability +- authenticate `/parse` before admission so unauthenticated requests cannot + consume parser leases or use overload behavior as a capacity oracle +- acquire a non-waiting process-local lease before the multipart body, + temporary-file allocation, PDF validation, or MinerU execution +- return fixed `429 Too Many Requests`, exact `Retry-After: 1`, and no-store + cache headers when every process lease is in use +- release admission leases after success, validation failure, backend failure, + unhandled exception, and request cancellation +- keep `NEWSDOM_MAX_CONCURRENT_PARSES` immutable, bounded to `1..128`, and + explicit in production deployment examples +- treat the process boundary as a last-resort resource control; retain + gateway-level per-tenant rate limits, quotas, and cluster resource limits +- do not add an unbounded in-process request or upload queue +- confirm docs endpoints remain informational and do not imply unsupported + authentication, execution, queueing, or durability guarantees ## Verification expectations -- unit/integration tests for parsing and error handling -- live localhost smoke for `/health`, `/docs`, and `/redoc` when docs +- unit and integration tests for parsing, authentication, readiness, overload, + cancellation, and sanitized error handling +- a realistic concurrent burst proving active parser count never exceeds the + configured process budget and excess work never reaches multipart parsing +- OpenAPI evidence for 401, 413, 415, 422, 429, 502, and 503 responses +- live localhost smoke for `/health`, `/ready`, `/docs`, and `/redoc` when docs or screenshots change -- PR/workflow review whenever GitHub Actions, release, or code-scanning posture changes +- deployment smoke that holds every lease, verifies one fixed pre-body 429, + releases one request, and proves capacity recovery +- resource evidence for any capacity increase: peak RSS/VRAM, temporary storage, + parser latency, error rate, replica count, and rollback threshold +- PR and workflow review whenever GitHub Actions, release, or code-scanning + posture changes diff --git a/manual/index.md b/manual/index.md index b808afa1..e221e7d6 100644 --- a/manual/index.md +++ b/manual/index.md @@ -1,40 +1,51 @@ -# NewsDOM API 개요 및 아키텍처 +# NewsDOM API -[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/Seongho-Bae/newsdom-api/badge)](https://securityscorecards.dev/viewer/?uri=github.com/Seongho-Bae/newsdom-api) +[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/ContextualWisdomLab/newsdom-api/badge)](https://securityscorecards.dev/viewer/?uri=github.com/ContextualWisdomLab/newsdom-api) +[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/ContextualWisdomLab/newsdom-api) -**NewsDOM API**는 스캔된 일본어 신문 PDF 문서를 분석하여, 웹 브라우저의 DOM(Document Object Model)과 유사한 기사(Article) 단위의 트리 구조로 파싱해주는 API 서비스입니다. +**NewsDOM API**는 PDF를 페이지·섹션·제목·본문 블록·이미지·캡션·바운딩 박스가 있는 안정적인 DOM형 JSON으로 정규화하는 API boundary입니다. 호출자는 특정 parser 내부 구조를 해석하는 대신 NewsDOM의 HTTP·schema·안전 계약을 사용합니다. -과거의 신문 이미지는 단순 텍스트 추출(OCR)만으로는 다단 레이아웃이나 이미지/캡션, 기사별 흐름을 파악하기 매우 어렵습니다. 본 프로젝트는 딥러닝 기반 레이아웃 분석 도구인 **MinerU**를 백엔드로 사용하여 이 문제를 해결합니다. +일본어 신문 OCR은 지원해 온 사례 중 하나이지만 제품 범위는 특정 언어 또는 강제 OCR에 한정되지 않습니다. 현재 API 계약은 `language` 선택과 `mode=auto | ocr | txt`를 구분하고, `/parse`의 fail-closed 인증과 `/health`·`/ready`의 서로 다른 의미를 유지합니다. ---- +## 현재 commercial parser 상태 -## ⚙️ 시스템 내부 아키텍처 (Under the Hood) +NewsDOM 자체 소스는 MIT입니다. 그러나 현재 소스의 legacy MinerU adapter가 사용하는 MinerU 3.x는 Apache-2.0에 추가 commercial 조건을 둔 MinerU Open Source License를 사용하므로 ContextualWisdomLab의 unrestricted commercial-inbound 정책과 호환되지 않습니다. -사용자가 API를 통해 PDF 파일을 업로드하면, NewsDOM 내부에서는 다음의 세 단계를 거쳐 데이터를 처리합니다: +따라서 MinerU package 설치, customer-supplied MinerU binary, `Dockerfile.nvidia` 기반 번들 배포를 **승인된 상업 경로로 안내하지 않습니다**. 대체 parser boundary와 검증 기준은 [#671](https://github.com/ContextualWisdomLab/newsdom-api/issues/671)에서 관리합니다. 승인된 parser가 없을 때 `/ready`가 fail-closed인 것은 정상입니다. -### 1. 서비스 래퍼 레이어 (`src/newsdom_api/service.py`) -FastAPI 엔드포인트(`/parse`)가 `UploadFile`로 전달받은 바이너리 데이터를 임시 디렉토리(Temporary Directory)에 저장한 후, 파이프라인 러너를 호출합니다. +## 제품 경계 -### 2. MinerU 파이프라인 러너 (`src/newsdom_api/mineru_runner.py`) -저장된 PDF를 대상으로 Python `subprocess` 모듈을 이용해 **MinerU CLI**를 백그라운드에서 실행합니다. 실제 내부적으로 실행되는 명령어는 다음과 같습니다: +- `POST /parse`는 PDF 업로드와 canonical NewsDOM response 계약을 소유합니다. +- `language`과 `mode`는 request compatibility surface이며 parser implementation detail과 분리되어야 합니다. +- `/parse`는 production profile에서 fail-closed Bearer 인증을 요구합니다. +- `GET /health`는 process liveness, `GET /ready`는 실제 traffic readiness입니다. +- parser admission은 process별 bounded non-waiting lease를 사용하며 포화 시 body 처리 전에 `429`로 거부합니다. +- 임시 workspace, parser failure 분류, schema validation, fixture provenance는 NewsDOM이 소유합니다. -```bash -mineru -p <업로드된_PDF> -o <임시출력경로> -b pipeline -m ocr -l japan +## 요청 흐름 + +```text +client + -> FastAPI request/auth boundary + -> bounded parser admission + -> request-scoped temporary PDF workspace + -> approved parser adapter port + -> NewsDOM normalization + schema validation + -> canonical JSON response ``` -> *참고: 일본어 신문 처리에 최적화하기 위해 `-l japan` (Language: Japanese) 옵션과 OCR 파이프라인 모드가 하드코딩되어 있습니다.* -명령어 실행이 완료되면 러너는 생성된 출력 폴더(OCR 하위 폴더)를 뒤져 `*_content_list.json` 파일과 `*_model.json` 결과물을 메모리로 로드합니다. +현재 legacy adapter는 교체 대상입니다. downstream consumer는 parser-specific type이나 model identity가 아니라 NewsDOM contract에 의존해야 합니다. + +## 시작하기 -### 3. DOM 빌더 (`src/newsdom_api/dom_builder.py`) -MinerU가 뱉어낸 선형적인(Linear) 블록 리스트(`content_list.json`)를 순회하면서 논리적인 트리 형태의 **`ParseResponse` (Canonical JSON)**로 재구성합니다. -- `role == "header"` 이면 상단 머릿말로(`PageNode.headers`) 분류 -- `type == "ad"` 또는 `role == "ad"` 이면 지면 광고(`PageNode.ads`)로 분류 -- `text_level == 1` 이거나 `role == "section_headings"`인 경우 새로운 기사의 시작(Headline)으로 인식하여 새 `ArticleNode`를 생성 -- 이후 등장하는 일반 텍스트는 해당 기사의 `body_blocks` 배열에 추가 -- `type == "image"` 이면 `ImageNode`를 생성하고 포함된 캡션 배열을 파싱하여 기사에 종속시킴 +- [설치 가이드](installation.md) — NewsDOM 자체의 lockfile 기반 개발/검증 환경과 parser license boundary. +- [사용 방법 및 API](api-reference.md) — `/parse`, language/mode, 인증, 응답과 오류 계약. +- [개발 및 기여](development.md) — 로컬 검증과 기여 흐름. +- [GitHub 저장소](https://github.com/ContextualWisdomLab/newsdom-api) — README, architecture, security, releases, source history. +- [Commercial parser replacement #671](https://github.com/ContextualWisdomLab/newsdom-api/issues/671) — 현재 상업 라이선스 차단과 replacement acceptance. -이러한 세밀한 내부 변환 과정을 통해 단순한 OCR 텍스트 덤프가 아닌, 프론트엔드에서 즉시 렌더링이 가능한 **구조화된 DOM 데이터**가 최종 반환됩니다. +## 릴리스와 공개 문서 ---- +`pyproject.toml`의 version은 source metadata이며 그 자체로 immutable release evidence가 아닙니다. 실제 릴리스는 [GitHub Releases](https://github.com/ContextualWisdomLab/newsdom-api/releases)와 해당 exact-head artifact/provenance evidence로 확인합니다. -👉 **[설치 가이드](installation.md)**를 읽고 직접 환경을 구성해 보세요. \ No newline at end of file +이 `manual/` source가 병합되었다는 사실만으로 공개 사이트 배포가 완료된 것은 아닙니다. GitHub Pages workflow 성공과 live HTTPS content 재확인이 필요합니다. diff --git a/manual/installation.md b/manual/installation.md index 754327de..45427c7c 100644 --- a/manual/installation.md +++ b/manual/installation.md @@ -1,87 +1,66 @@ # 시스템 환경 및 설치 가이드 -이 문서에서는 NewsDOM API를 로컬 환경에 설치하는 방법을 안내합니다. +이 문서는 **NewsDOM API 자체**를 개발·검증하기 위한 설치 절차를 설명합니다. 현재 소스의 legacy MinerU adapter는 별도 parser runtime을 호출하지만, MinerU 3.x의 추가 상업 라이선스 조건은 ContextualWisdomLab의 unrestricted commercial-inbound 정책과 호환되지 않습니다. 따라서 이 문서는 MinerU 설치를 지원 경로로 안내하지 않습니다. 대체 parser boundary는 [#671](https://github.com/ContextualWisdomLab/newsdom-api/issues/671)에서 추적합니다. -## 🛠️ 시스템 요구사항 +## 시스템 요구사항 - **Python**: Required: `>=3.10, <3.14` -- **운영체제**: Linux 또는 macOS 권장 (윈도우의 경우 WSL2 사용 권장) -- **하드웨어 (GPU)**: `MinerU` 딥러닝 기반 파이프라인을 구동하기 위해서는 최소 **8GB 이상의 RAM**이 필요하며, 실시간 처리를 위해 **NVIDIA GPU(CUDA 11.x/12.x 호환)** 및 `PyTorch` 환경이 권장됩니다. -- **의존성 (Python)**: - - `fastapi>=0.115,<1.0`, `uvicorn>=0.30,<1.0`, `pydantic>=2.9,<3.0` - - `python-multipart`, `reportlab`, `Pillow`, `pypdf` 등 +- **운영체제**: Linux, macOS, Windows의 지원 Python 환경 +- **패키지 관리자**: `uv` +- **NewsDOM 의존성**: `pyproject.toml` / `uv.lock`이 권위 있는 설치 계약 ---- +GPU, OCR model, parser model weight는 NewsDOM 자체 설치 요구사항으로 간주하지 않습니다. 승인된 parser backend가 도입되면 해당 backend의 플랫폼·리소스·라이선스 요구사항을 별도 profile로 문서화해야 합니다. -## 1. 기본 테스트 및 개발 모드 설치 +## 1. 저장소 환경 설치 -가장 간단한 형태로 파이썬 가상환경(Virtual Environment)을 생성하고 패키지를 설치합니다. 이 모드에서는 실제 `MinerU` 모델이 로드되지 않으며, `pytest`나 합성 픽스처(Synthetic Fixtures) 기반 테스트 용도로 적합합니다. 예시 명령은 `python3.10`을 사용하지만, 지원 범위 안의 다른 인터프리터도 동일하게 사용할 수 있습니다. +예시는 `python3.10`을 사용하지만, 지원 범위 안의 다른 인터프리터도 사용할 수 있습니다. `uv`를 이용한 canonical setup은 다음과 같습니다. ```bash -# 가상 환경 생성 (권장 예시: python3.10) -python3.10 -m venv .venv +uv sync --frozen --all-extras +``` -# 가상 환경 활성화 -# macOS / Linux -source .venv/bin/activate -# Windows (WSL 환경 제외) -# .venv\Scripts\activate +동일한 가상환경 위치를 직접 확인해야 하는 경우 macOS/Linux에서는 `.venv/bin/python`, Windows에서는 `.venv\Scripts\python.exe`를 사용합니다. -# pip 업그레이드 -python -m pip install --upgrade pip +과거 수동 설치 예시인 `python3.10 -m venv .venv`와 `pip install -e ".[dev]"`는 호환성 참고용일 뿐 canonical repository setup이 아닙니다. 현재 검증과 CI는 lockfile 기반 `uv` 환경을 기준으로 합니다. -# 의존성 패키지와 함께 개발 모드로 설치 -pip install -e ".[dev]" -``` +## 2. parser runtime 정책 ---- +현재 repository source에는 legacy MinerU adapter가 남아 있지만 다음 경로는 **상업 배포 승인 경로가 아닙니다**. -## 2. MinerU 백엔드 포함 실제 파싱 모드 설치 +- MinerU Python package 직접 설치 +- `NEWSDOM_MINERU_BIN`으로 고객이 공급한 MinerU binary 연결 +- `Dockerfile.nvidia`를 이용한 MinerU-bundled image 빌드/배포 +- MinerU를 별도 container/process/service로 옮긴 뒤 NewsDOM의 필수 runtime으로 사용하는 구성 -`MinerU` 백엔드를 사용하여 실제 스캔된 일본어 신문 PDF 파싱 작업을 수행하려면 MinerU CLI를 별도로 설치해야 합니다. +이 제한은 단순 attribution 문제가 아니라 upstream의 추가 commercial threshold 조건과 조직의 inbound 정책 차이 때문입니다. MIT로 배포되는 NewsDOM 자체가 제3자 parser를 재라이선스하지 않습니다. -```bash -# MinerU 파이프라인 CLI 설치 -pip install "mineru[pipeline]==3.4.4" -``` +승인된 parser backend가 아직 없는 환경에서 `/ready`가 실패하는 것은 올바른 fail-closed 상태입니다. parser를 우회하거나 readiness를 강제로 green으로 만들지 마십시오. -이 명령어를 통해 **`mineru[pipeline]==3.4.4`** 버전이 설치되며 딥러닝 기반 모델을 위한 준비가 완료됩니다. 설치 후 처음 API 서버를 구동하고 PDF를 파싱할 때 모델(Weight) 파일을 백그라운드에서 다운로드할 수 있으므로, 첫 실행에는 다운로드 대기 시간이 발생할 수 있습니다. +## 3. 테스트와 API shell 확인 -### 커스텀 MinerU 실행 경로 (고급) -만약 `mineru` CLI 바이너리가 시스템 PATH에 잡혀있지 않거나, 특정 가상환경의 실행 파일을 수동으로 지정하고 싶다면 환경변수를 설정하세요: +저장소 테스트를 실행합니다. ```bash -# newsdom_api/mineru_runner.py 에서 이 환경변수를 우선 탐색합니다. -export NEWSDOM_MINERU_BIN="/path/to/custom/mineru" +uv run pytest ``` ---- - -## 3. 설치 확인 및 상태 점검 - -설치가 정상적으로 완료되었는지 확인하려면 기본 테스트를 구동해보세요. +API shell은 parser 설치 없이도 liveness, 인증 설정, 요청 계약을 개발·검증하는 데 사용할 수 있습니다. ```bash -# 파이썬 경고(Warning)를 에러로 취급하여 꼼꼼하게 검사 -PYTHONWARNINGS=error pytest +uv run uvicorn --app-dir src newsdom_api.main:app --host 127.0.0.1 --port 8000 --reload ``` -현재 저장소에는 별도의 `integration` 마커 테스트 묶음이 없으므로, -설치 확인의 기준은 기본 `pytest` 스위트 통과입니다. 추가로 MinerU -경로와 API 동작까지 확인하려면 서버를 직접 띄운 뒤 수동 API 점검 -단계를 수행하세요. +다른 터미널에서 liveness를 확인합니다. ```bash -# 별도 터미널에서 API 서버 기동 -python -m uvicorn --app-dir src newsdom_api.main:app --host 0.0.0.0 --port 8000 --reload - -# 다른 터미널에서 상태 확인 curl -sS http://127.0.0.1:8000/health ``` -정상 응답 예시는 `{"status": "ok"}`이며 HTTP 200 상태 코드를 반환해야 합니다. +정상 프로세스는 HTTP 200을 반환합니다. `/health` 성공은 parser readiness를 의미하지 않습니다. 실제 트래픽 라우팅 판단에는 `/ready`를 사용하고, 승인된 parser가 없는 동안에는 fail-closed 상태를 유지합니다. -모든 테스트(`tests/`)가 성공적으로 통과했다면 API 서버를 실행할 -준비가 된 것입니다. +## 4. 다음 단계 -👉 다음 단계: **[API 레퍼런스 및 사용 방법](api-reference.md)** +- [API 레퍼런스 및 사용 방법](api-reference.md) +- [개발 및 기여](development.md) +- [Commercial parser replacement #671](https://github.com/ContextualWisdomLab/newsdom-api/issues/671) +- [GitHub repository README](https://github.com/ContextualWisdomLab/newsdom-api) diff --git a/mkdocs.yml b/mkdocs.yml index e2489723..4649b793 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,8 +1,8 @@ site_name: NewsDOM API 매뉴얼 -site_url: https://seongho-bae.github.io/newsdom-api/ -site_description: 일본어 신문 스캔 PDF를 DOM 형태의 트리로 파싱하는 API -repo_url: https://github.com/Seongho-Bae/newsdom-api -repo_name: Seongho-Bae/newsdom-api +site_url: https://contextualwisdomlab.github.io/newsdom-api/ +site_description: 다국어 PDF를 구조화된 DOM형 JSON으로 변환하는 NewsDOM API 문서 +repo_url: https://github.com/ContextualWisdomLab/newsdom-api +repo_name: ContextualWisdomLab/newsdom-api docs_dir: manual diff --git a/src/newsdom_api/admission.py b/src/newsdom_api/admission.py new file mode 100644 index 00000000..72fe47de --- /dev/null +++ b/src/newsdom_api/admission.py @@ -0,0 +1,31 @@ +"""Application-local concurrency state for parser work.""" + +from threading import BoundedSemaphore + + +class ParseAdmissionLimiter: + """Bound concurrent parser work without queueing excess requests.""" + + __slots__ = ("_capacity", "_semaphore") + + def __init__(self, capacity: int) -> None: + """Create a limiter with an immutable number of parser leases.""" + + self._capacity = capacity + self._semaphore = BoundedSemaphore(capacity) + + @property + def capacity(self) -> int: + """Return the immutable number of parser leases.""" + + return self._capacity + + def try_acquire(self) -> bool: + """Acquire one parser lease immediately or report saturation.""" + + return self._semaphore.acquire(blocking=False) + + def release(self) -> None: + """Return one previously acquired parser lease.""" + + self._semaphore.release() diff --git a/src/newsdom_api/config.py b/src/newsdom_api/config.py index 34bd05f9..458b39ad 100644 --- a/src/newsdom_api/config.py +++ b/src/newsdom_api/config.py @@ -10,7 +10,10 @@ API_TOKEN_ENV_VAR = "NEWSDOM_API_TOKEN" AUTH_MODE_ENV_VAR = "NEWSDOM_AUTH_MODE" RUNTIME_PROFILE_ENV_VAR = "NEWSDOM_RUNTIME_PROFILE" +MAX_CONCURRENT_PARSES_ENV_VAR = "NEWSDOM_MAX_CONCURRENT_PARSES" MAX_BEARER_HEADER_BYTES = 4096 +DEFAULT_MAX_CONCURRENT_PARSES = 1 +MAX_MAX_CONCURRENT_PARSES = 128 class RuntimeConfigurationError(ValueError): @@ -31,6 +34,20 @@ class RuntimeProfile(str, Enum): DEVELOPMENT = "development" +def _validated_parse_capacity(value: object) -> int: + """Return one supported non-boolean per-process parse capacity.""" + + if ( + isinstance(value, bool) + or not isinstance(value, int) + or not 1 <= value <= MAX_MAX_CONCURRENT_PARSES + ): + raise RuntimeConfigurationError( + "Invalid value for NEWSDOM_MAX_CONCURRENT_PARSES; use an integer from 1 to 128" + ) + return value + + @dataclass(frozen=True, slots=True) class RuntimeSettings: """Validated settings frozen for the lifetime of one FastAPI application.""" @@ -38,6 +55,7 @@ class RuntimeSettings: authentication_mode: AuthenticationMode = AuthenticationMode.REQUIRED runtime_profile: RuntimeProfile = RuntimeProfile.PRODUCTION api_token: str | None = field(default=None, repr=False) + max_concurrent_parses: int = DEFAULT_MAX_CONCURRENT_PARSES def __post_init__(self) -> None: """Normalize secrets once and reject unsafe direct construction.""" @@ -50,6 +68,12 @@ def __post_init__(self) -> None: "Authentication can be disabled only in the development runtime profile" ) + object.__setattr__( + self, + "max_concurrent_parses", + _validated_parse_capacity(self.max_concurrent_parses), + ) + if self.api_token is None: return @@ -98,6 +122,22 @@ def _parse_enum_value( return raw +def _parse_capacity_value(source: Mapping[str, str]) -> int: + """Parse the bounded parser budget without echoing malformed input.""" + + raw = source.get( + MAX_CONCURRENT_PARSES_ENV_VAR, + str(DEFAULT_MAX_CONCURRENT_PARSES), + ).strip() + try: + parsed = int(raw) + except ValueError as exc: + raise RuntimeConfigurationError( + "Invalid value for NEWSDOM_MAX_CONCURRENT_PARSES; use an integer from 1 to 128" + ) from exc + return _validated_parse_capacity(parsed) + + def get_api_token(source: Mapping[str, str] | None = None) -> str | None: """Return a normalized bootstrap token without logging or exposing it.""" @@ -136,4 +176,5 @@ def load_runtime_settings( authentication_mode=authentication_mode, runtime_profile=runtime_profile, api_token=get_api_token(values), + max_concurrent_parses=_parse_capacity_value(values), ) diff --git a/src/newsdom_api/main.py b/src/newsdom_api/main.py index f61aafc2..ec6dd48b 100644 --- a/src/newsdom_api/main.py +++ b/src/newsdom_api/main.py @@ -24,6 +24,7 @@ from pypdf import PdfReader from pypdf.errors import PdfReadError +from .admission import ParseAdmissionLimiter from .config import ( AuthenticationMode, MAX_BEARER_HEADER_BYTES, @@ -47,7 +48,9 @@ PAYLOAD_TOO_LARGE_DETAIL = "Payload Too Large" INVALID_PARSE_PARAMS_DETAIL = "Invalid parse parameters" UNAUTHORIZED_DETAIL = "Unauthorized" +TOO_MANY_REQUESTS_DETAIL = "Too Many Requests" SERVICE_UNAVAILABLE_DETAIL = "Service Unavailable" +RETRY_AFTER_SECONDS = "1" LOGGER = logging.getLogger("newsdom_api") BEARER_SCHEME = HTTPBearer(auto_error=False, scheme_name="BearerAuth") @@ -108,6 +111,16 @@ def _unauthorized_response() -> JSONResponse: ) +def _too_many_requests_response() -> JSONResponse: + """Return one fixed response for a saturated parser process.""" + + return JSONResponse( + status_code=429, + content={"detail": TOO_MANY_REQUESTS_DETAIL}, + headers={"Retry-After": RETRY_AFTER_SECONDS}, + ) + + def _parse_access_failure(request: Request) -> JSONResponse | None: """Validate `/parse` authorization before multipart upload parsing begins.""" @@ -140,12 +153,22 @@ async def security_boundary_middleware( request: Request, call_next: Callable, ) -> Response: - """Enforce parser authorization before reading the request body and add headers.""" + """Enforce parser access boundaries before reading the request body.""" if request.method == "POST" and request.scope.get("path") == "/parse": failure = _parse_access_failure(request) if failure is not None: return _apply_security_headers(failure, request) + + limiter: ParseAdmissionLimiter = request.app.state.parse_admission_limiter + if not limiter.try_acquire(): + return _apply_security_headers(_too_many_requests_response(), request) + try: + response = await call_next(request) + finally: + limiter.release() + return _apply_security_headers(response, request) + response = await call_next(request) return _apply_security_headers(response, request) @@ -167,8 +190,18 @@ def health() -> HealthResponse: return HealthResponse() +def approved_parser_runtime_available() -> bool: + """Fail closed until an unrestricted parser backend is released.""" + + if not mineru_runtime_available(): + return False + # The discoverable MinerU 3.x adapter is legacy-only and is not approved + # for the unrestricted commercial runtime tracked by issue #671. + return False + + def ready(request: Request) -> ReadinessResponse: - """Return readiness only when authentication and MinerU runtime are available.""" + """Return readiness only for valid authentication and an approved parser.""" settings = _runtime_settings(request) runtime_probe = request.app.state.runtime_readiness_probe @@ -324,8 +357,11 @@ def create_app( }, ) application.state.runtime_settings = application_settings + application.state.parse_admission_limiter = ParseAdmissionLimiter( + application_settings.max_concurrent_parses + ) application.state.runtime_readiness_probe = ( - runtime_readiness_probe or mineru_runtime_available + runtime_readiness_probe or approved_parser_runtime_available ) application.middleware("http")(security_boundary_middleware) application.add_exception_handler(Exception, global_exception_handler) @@ -346,7 +382,7 @@ def create_app( summary="Readiness Check", description=( "Returns ready only when parser authentication configuration and " - "the MinerU runtime can safely accept traffic." + "an approved parser backend can safely accept traffic." ), responses={503: {"description": SERVICE_UNAVAILABLE_DETAIL}}, tags=["System"], @@ -367,6 +403,7 @@ def create_app( 413: {"description": PAYLOAD_TOO_LARGE_DETAIL}, 415: {"description": UNSUPPORTED_MEDIA_DETAIL}, 422: {"description": INVALID_PARSE_PARAMS_DETAIL}, + 429: {"description": TOO_MANY_REQUESTS_DETAIL}, 502: {"description": "Bad Gateway"}, 503: {"description": SERVICE_UNAVAILABLE_DETAIL}, }, @@ -375,4 +412,4 @@ def create_app( return application -app = create_app() +app = create_app() \ No newline at end of file diff --git a/tests/test_auth.py b/tests/test_auth.py index 3dc8311b..83db35f7 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -38,6 +38,7 @@ def _settings( token: str | None = "s3cret-token", mode: AuthenticationMode = AuthenticationMode.REQUIRED, profile: RuntimeProfile = RuntimeProfile.PRODUCTION, + max_concurrent_parses: int = 1, ) -> RuntimeSettings: """Build one explicit immutable settings object for an application test.""" @@ -45,6 +46,7 @@ def _settings( authentication_mode=mode, runtime_profile=profile, api_token=token, + max_concurrent_parses=max_concurrent_parses, ) @@ -302,6 +304,20 @@ def test_default_runtime_probe_is_not_an_unconditional_success( assert response.status_code == 503 +def test_legacy_mineru_executable_is_not_an_approved_parser( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A discoverable legacy executable cannot make commercial traffic ready.""" + + monkeypatch.setattr( + "newsdom_api.main.mineru_runtime_available", lambda: True, raising=False + ) + response = TestClient(create_app(_settings())).get("/ready") + + assert response.status_code == 503 + assert response.json() == {"detail": "Service Unavailable"} + + def test_authentication_middleware_rejects_before_reading_request_body() -> None: """Reject parser access before ASGI body receive or downstream parsing.""" @@ -346,7 +362,8 @@ def test_concurrent_requests_cannot_switch_authentication_state( """Concurrent callers should observe one frozen token and mode.""" application = create_app( - _settings(token="fixed"), runtime_readiness_probe=lambda: True + _settings(token="fixed", max_concurrent_parses=20), + runtime_readiness_probe=lambda: True, ) def request(token: str) -> int: @@ -388,22 +405,80 @@ def test_get_api_token_normalizes_bootstrap_transport() -> None: assert get_api_token({}) is None assert get_api_token({API_TOKEN_ENV_VAR: " "}) is None - assert get_api_token({API_TOKEN_ENV_VAR: " padded-token\n"}) == "padded-token" + assert get_api_token({API_TOKEN_ENV_VAR: " mounted-secret\n"}) == "mounted-secret" + +def test_environment_settings_are_snapshotted_once( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Loading settings once should isolate them from later environment mutation.""" -def test_default_runtime_settings_are_required_production() -> None: - """Missing mode variables must select the fail-closed production contract.""" + environment = { + AUTH_MODE_ENV_VAR: "required", + RUNTIME_PROFILE_ENV_VAR: "production", + API_TOKEN_ENV_VAR: "initial-token", + } + settings = load_runtime_settings(environment) + environment[API_TOKEN_ENV_VAR] = "changed-token" + monkeypatch.setenv(API_TOKEN_ENV_VAR, "process-token") - settings = load_runtime_settings({API_TOKEN_ENV_VAR: "token"}) + assert settings.api_token == "initial-token" - assert settings.authentication_mode is AuthenticationMode.REQUIRED - assert settings.runtime_profile is RuntimeProfile.PRODUCTION - assert settings.authentication_ready is True +def test_default_factory_loads_settings_once_per_application( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The factory should evaluate runtime configuration only during app creation.""" + + calls = {"count": 0} + frozen = _settings() + + def fake_loader(): + calls["count"] += 1 + return frozen + + monkeypatch.setattr("newsdom_api.main.load_runtime_settings", fake_loader) + application = create_app(runtime_readiness_probe=lambda: True) + + assert calls["count"] == 1 + assert application.state.runtime_settings is frozen + + +def test_compatibility_get_api_token_reads_process_environment( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The compatibility helper should keep its explicit process lookup contract.""" + + monkeypatch.setenv(API_TOKEN_ENV_VAR, "compat-token") + + assert get_api_token() == "compat-token" + + +def test_security_headers_are_applied_to_unhandled_server_errors() -> None: + """Unhandled errors should keep the same response hardening as normal traffic.""" -def test_config_module_exposes_versioned_environment_contract() -> None: - """Deployment tooling should use the exact documented environment names.""" + async def failing_route() -> None: + raise RuntimeError("internal path /tmp/private-token") - assert config.API_TOKEN_ENV_VAR == "NEWSDOM_API_TOKEN" - assert config.AUTH_MODE_ENV_VAR == "NEWSDOM_AUTH_MODE" - assert config.RUNTIME_PROFILE_ENV_VAR == "NEWSDOM_RUNTIME_PROFILE" + application = create_app(_settings(), runtime_readiness_probe=lambda: True) + application.add_api_route("/boom", failing_route, methods=["GET"]) + response = TestClient(application, raise_server_exceptions=False).get("/boom") + + assert response.status_code == 500 + assert response.json() == {"detail": "Internal Server Error"} + assert response.headers["X-Content-Type-Options"] == "nosniff" + assert "private-token" not in response.text + + +def test_main_module_logs_missing_required_token_without_crashing( + caplog: pytest.LogCaptureFixture, +) -> None: + """Application creation should expose configuration failure only to operators.""" + + caplog.set_level(logging.ERROR, logger="newsdom_api") + create_app(_settings(token=None), runtime_readiness_probe=lambda: True) + + assert "Parser authentication configuration is unavailable" in caplog.text + assert "token" not in caplog.text.lower().replace( + "parser authentication configuration is unavailable", "" + ) diff --git a/tests/test_changelog.py b/tests/test_changelog.py index 64eb9985..24d18095 100644 --- a/tests/test_changelog.py +++ b/tests/test_changelog.py @@ -27,11 +27,11 @@ def test_initial_release_entry_keeps_added_section_and_links(): in text ) assert ( - "[Unreleased]: https://github.com/Seongho-Bae/newsdom-api/compare/v0.2.0...HEAD" + "[Unreleased]: https://github.com/ContextualWisdomLab/newsdom-api/compare/v0.2.0...HEAD" in text ) assert ( - "[0.1.0]: https://github.com/Seongho-Bae/newsdom-api/releases/tag/v0.1.0" + "[0.1.0]: https://github.com/ContextualWisdomLab/newsdom-api/releases/tag/v0.1.0" in text ) @@ -42,10 +42,10 @@ def test_changelog_prepares_the_0_1_1_release_entry(): assert "### Added" in text assert "*.intoto.jsonl" in text assert ( - "[Unreleased]: https://github.com/Seongho-Bae/newsdom-api/compare/v0.2.0...HEAD" + "[Unreleased]: https://github.com/ContextualWisdomLab/newsdom-api/compare/v0.2.0...HEAD" in text ) assert ( - "[0.1.1]: https://github.com/Seongho-Bae/newsdom-api/compare/v0.1.0...v0.1.1" + "[0.1.1]: https://github.com/ContextualWisdomLab/newsdom-api/compare/v0.1.0...v0.1.1" in text ) diff --git a/tests/test_docker_delivery.py b/tests/test_docker_delivery.py index 44018975..ebf6312a 100644 --- a/tests/test_docker_delivery.py +++ b/tests/test_docker_delivery.py @@ -169,9 +169,10 @@ def test_readme_describes_default_image_as_api_only_runtime() -> None: assert "does not bundle the MinerU runtime" in text assert "real `/parse` execution" not in text assert ( - "requires a compatible MinerU runtime to be available inside the container image" + "`/ready` must stay fail-closed until an approved parser backend exists" in text ) + assert "commercially blocked under organization policy" in text def test_docker_command_matchers_allow_wrapped_whitespace(): diff --git a/tests/test_manual_docs.py b/tests/test_manual_docs.py index aa72a2da..af875650 100644 --- a/tests/test_manual_docs.py +++ b/tests/test_manual_docs.py @@ -50,23 +50,26 @@ def test_development_doc_adds_blank_line_after_branch_rules_heading() -> None: assert _has_blank_line_after_heading(text, "### 🌿 브랜치 규칙") -def test_installation_doc_uses_quoted_extras_and_clear_python_wording(): +def test_installation_doc_uses_locked_repo_setup_and_blocks_restricted_parser_install(): text = Path("manual/installation.md").read_text(encoding="utf-8") assert "Required: `>=3.10, <3.14`" in text for token in ("python3.10", "지원 범위", "인터프리터"): assert token in text assert "python3.10 -m venv .venv" in text assert 'pip install -e ".[dev]"' in text - assert 'pip install "mineru[pipeline]==3.4.4"' in text + assert "uv sync --frozen --all-extras" in text + assert 'pip install "mineru[pipeline]==3.4.4"' not in text + assert "#671" in text + assert "상업 배포 승인 경로가 아닙니다" in text def test_installation_doc_includes_manual_api_healthcheck_commands(): text = Path("manual/installation.md").read_text(encoding="utf-8") for token in [ - "python -m uvicorn", + "uv run uvicorn", "--app-dir src", "newsdom_api.main:app", - "--host 0.0.0.0", + "--host 127.0.0.1", "--port 8000", "--reload", ]: diff --git a/tests/test_parse_admission.py b/tests/test_parse_admission.py new file mode 100644 index 00000000..af262a8f --- /dev/null +++ b/tests/test_parse_admission.py @@ -0,0 +1,313 @@ +"""Contracts for process-local parser admission control.""" + +import asyncio +import json + +import pytest +from fastapi import FastAPI, Request, Response +from fastapi.responses import JSONResponse + +from newsdom_api.config import AuthenticationMode, RuntimeProfile, RuntimeSettings +from newsdom_api.main import create_app, security_boundary_middleware + + +def _development_settings(*, capacity: int) -> RuntimeSettings: + """Return explicit development settings for one admission-control test.""" + + return RuntimeSettings( + authentication_mode=AuthenticationMode.DISABLED, + runtime_profile=RuntimeProfile.DEVELOPMENT, + max_concurrent_parses=capacity, + ) + + +def _production_settings(*, capacity: int) -> RuntimeSettings: + """Return fail-closed production settings with one fixed parser token.""" + + return RuntimeSettings( + authentication_mode=AuthenticationMode.REQUIRED, + runtime_profile=RuntimeProfile.PRODUCTION, + api_token="admission-test-token", + max_concurrent_parses=capacity, + ) + + +def _parse_request( + application: FastAPI, + *, + headers: list[tuple[bytes, bytes]] | None = None, +) -> Request: + """Build a body-hostile parse request for direct middleware verification.""" + + async def receive() -> dict[str, object]: + raise AssertionError("admission middleware must not read the request body") + + return Request( + { + "type": "http", + "asgi": {"version": "3.0", "spec_version": "2.3"}, + "http_version": "1.1", + "method": "POST", + "scheme": "http", + "path": "/parse", + "raw_path": b"/parse", + "query_string": b"", + "headers": headers or [], + "client": ("testclient", 50000), + "server": ("testserver", 80), + "root_path": "", + "app": application, + }, + receive, + ) + + +def test_application_binds_configured_process_local_parse_capacity() -> None: + """Each application should own the immutable parse capacity it was given.""" + + application = create_app( + _development_settings(capacity=3), + runtime_readiness_probe=lambda: True, + ) + + assert application.state.parse_admission_limiter.capacity == 3 + + +def test_limiter_rejects_excess_work_without_waiting_and_recovers() -> None: + """One released lease should immediately restore one unit of capacity.""" + + application = create_app( + _development_settings(capacity=1), + runtime_readiness_probe=lambda: True, + ) + limiter = application.state.parse_admission_limiter + + assert limiter.try_acquire() is True + assert limiter.try_acquire() is False + + limiter.release() + + assert limiter.try_acquire() is True + limiter.release() + + +def test_applications_do_not_share_parser_leases() -> None: + """One saturated application must not reduce another application's capacity.""" + + first = create_app( + _development_settings(capacity=1), + runtime_readiness_probe=lambda: True, + ) + second = create_app( + _development_settings(capacity=1), + runtime_readiness_probe=lambda: True, + ) + + assert first.state.parse_admission_limiter.try_acquire() is True + assert first.state.parse_admission_limiter.try_acquire() is False + assert second.state.parse_admission_limiter.try_acquire() is True + + first.state.parse_admission_limiter.release() + second.state.parse_admission_limiter.release() + + +@pytest.mark.asyncio +async def test_saturation_returns_fixed_429_before_downstream_or_body_work() -> None: + """Excess parser work should fail before multipart or parser execution.""" + + application = create_app( + _development_settings(capacity=1), + runtime_readiness_probe=lambda: True, + ) + limiter = application.state.parse_admission_limiter + assert limiter.try_acquire() is True + downstream_called = False + + async def downstream(_request: Request) -> Response: + nonlocal downstream_called + downstream_called = True + return JSONResponse({"status": "unexpected"}) + + response = await security_boundary_middleware( + _parse_request(application), + downstream, + ) + + assert response.status_code == 429 + assert json.loads(response.body) == {"detail": "Too Many Requests"} + assert response.headers["Retry-After"] == "1" + assert response.headers["Cache-Control"] == "no-store, no-cache, max-age=0" + assert downstream_called is False + limiter.release() + + +@pytest.mark.asyncio +async def test_successful_downstream_work_returns_its_parser_lease() -> None: + """A completed parser request should make its lease immediately reusable.""" + + application = create_app( + _development_settings(capacity=1), + runtime_readiness_probe=lambda: True, + ) + limiter = application.state.parse_admission_limiter + + async def downstream(_request: Request) -> Response: + return JSONResponse({"status": "complete"}) + + response = await security_boundary_middleware( + _parse_request(application), + downstream, + ) + + assert response.status_code == 200 + assert limiter.try_acquire() is True + limiter.release() + + +@pytest.mark.asyncio +async def test_exceptional_downstream_work_returns_its_parser_lease() -> None: + """A parser exception must not permanently consume process capacity.""" + + application = create_app( + _development_settings(capacity=1), + runtime_readiness_probe=lambda: True, + ) + limiter = application.state.parse_admission_limiter + + async def downstream(_request: Request) -> Response: + raise RuntimeError("parser failed") + + with pytest.raises(RuntimeError, match="parser failed"): + await security_boundary_middleware( + _parse_request(application), + downstream, + ) + + assert limiter.try_acquire() is True + limiter.release() + + +@pytest.mark.asyncio +async def test_cancelled_downstream_work_returns_its_parser_lease() -> None: + """Request cancellation must not leak a parser lease.""" + + application = create_app( + _development_settings(capacity=1), + runtime_readiness_probe=lambda: True, + ) + limiter = application.state.parse_admission_limiter + + async def downstream(_request: Request) -> Response: + raise asyncio.CancelledError + + with pytest.raises(asyncio.CancelledError): + await security_boundary_middleware( + _parse_request(application), + downstream, + ) + + assert limiter.try_acquire() is True + limiter.release() + + +@pytest.mark.asyncio +async def test_authentication_failure_does_not_consume_parser_capacity() -> None: + """Unauthorized requests should be rejected before admission accounting.""" + + application = create_app( + _production_settings(capacity=2), + runtime_readiness_probe=lambda: True, + ) + downstream_called = False + + async def downstream(_request: Request) -> Response: + nonlocal downstream_called + downstream_called = True + return JSONResponse({"status": "unexpected"}) + + response = await security_boundary_middleware( + _parse_request(application), + downstream, + ) + limiter = application.state.parse_admission_limiter + + assert response.status_code == 401 + assert downstream_called is False + assert limiter.try_acquire() is True + assert limiter.try_acquire() is True + assert limiter.try_acquire() is False + limiter.release() + limiter.release() + + +@pytest.mark.asyncio +async def test_thirty_two_requests_observe_a_four_request_capacity_bound() -> None: + """A realistic burst should admit four requests and reject the other 28.""" + + capacity = 4 + application = create_app( + _development_settings(capacity=capacity), + runtime_readiness_probe=lambda: True, + ) + all_admitted = asyncio.Event() + release_admitted = asyncio.Event() + admitted_count = 0 + + async def held_downstream(_request: Request) -> Response: + nonlocal admitted_count + admitted_count += 1 + if admitted_count == capacity: + all_admitted.set() + await release_admitted.wait() + return JSONResponse({"status": "complete"}) + + async def unexpected_downstream(_request: Request) -> Response: + raise AssertionError("saturated requests must not reach downstream parsing") + + admitted_tasks = [ + asyncio.create_task( + security_boundary_middleware( + _parse_request(application), + held_downstream, + ) + ) + for _ in range(capacity) + ] + await asyncio.wait_for(all_admitted.wait(), timeout=1.0) + + saturated_responses = await asyncio.gather( + *[ + security_boundary_middleware( + _parse_request(application), + unexpected_downstream, + ) + for _ in range(32 - capacity) + ] + ) + + assert [response.status_code for response in saturated_responses] == [429] * 28 + assert all(response.headers["Retry-After"] == "1" for response in saturated_responses) + + release_admitted.set() + admitted_responses = await asyncio.gather(*admitted_tasks) + + assert [response.status_code for response in admitted_responses] == [200] * capacity + limiter = application.state.parse_admission_limiter + reacquired = [limiter.try_acquire() for _ in range(capacity)] + assert reacquired == [True] * capacity + assert limiter.try_acquire() is False + for _ in range(capacity): + limiter.release() + + +def test_openapi_documents_parser_saturation_response() -> None: + """The parser operation should publish the fixed 429 response contract.""" + + application = create_app( + _development_settings(capacity=1), + runtime_readiness_probe=lambda: True, + ) + + response_contract = application.openapi()["paths"]["/parse"]["post"]["responses"] + + assert response_contract["429"]["description"] == "Too Many Requests" diff --git a/tests/test_parse_capacity.py b/tests/test_parse_capacity.py new file mode 100644 index 00000000..8a381912 --- /dev/null +++ b/tests/test_parse_capacity.py @@ -0,0 +1,112 @@ +"""Contracts for bounded parser admission control.""" + +from pathlib import Path + +import pytest +import yaml + +from newsdom_api.config import ( + RuntimeConfigurationError, + RuntimeSettings, + load_runtime_settings, +) + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] + + +def _text(path: str) -> str: + """Read one repository-relative UTF-8 artifact.""" + + return (REPOSITORY_ROOT / path).read_text(encoding="utf-8") + + +def test_runtime_settings_default_to_one_concurrent_parse() -> None: + """A standalone process should admit one expensive parse by default.""" + + settings = RuntimeSettings() + + assert settings.max_concurrent_parses == 1 + + +def test_runtime_settings_load_configured_parse_capacity() -> None: + """Operators should be able to set a bounded per-process parse budget.""" + + settings = load_runtime_settings({"NEWSDOM_MAX_CONCURRENT_PARSES": "8"}) + + assert settings.max_concurrent_parses == 8 + + +@pytest.mark.parametrize("raw_value", ["", "0", "129", "not-an-integer"]) +def test_runtime_settings_reject_invalid_environment_capacity(raw_value: str) -> None: + """Environment input must be an integer in the supported process range.""" + + with pytest.raises(RuntimeConfigurationError, match="NEWSDOM_MAX_CONCURRENT_PARSES"): + load_runtime_settings({"NEWSDOM_MAX_CONCURRENT_PARSES": raw_value}) + + +@pytest.mark.parametrize("capacity", [0, 129, True, "3"]) +def test_runtime_settings_reject_invalid_direct_capacity(capacity: object) -> None: + """Direct construction must preserve the same immutable capacity boundary.""" + + with pytest.raises(RuntimeConfigurationError, match="NEWSDOM_MAX_CONCURRENT_PARSES"): + RuntimeSettings(max_concurrent_parses=capacity) # type: ignore[arg-type] + + +def test_deployment_examples_publish_the_per_process_capacity_budget() -> None: + """Every production example should expose one explicit parser capacity.""" + + assert "NEWSDOM_MAX_CONCURRENT_PARSES=1" in _text(".env.example") + + compose = yaml.safe_load(_text("docker-compose.yml")) + compose_environment = compose["services"]["newsdom-api"]["environment"] + assert compose_environment["NEWSDOM_MAX_CONCURRENT_PARSES"] == "1" + + documents = list( + yaml.safe_load_all(_text("docs/operations/kubernetes-deployment.yaml")) + ) + deployment = next(document for document in documents if document["kind"] == "Deployment") + container = deployment["spec"]["template"]["spec"]["containers"][0] + environment = {entry["name"]: entry for entry in container["env"]} + assert environment["NEWSDOM_MAX_CONCURRENT_PARSES"]["value"] == "1" + + +def test_operator_docs_explain_non_waiting_backpressure_and_replica_capacity() -> None: + """Operators need enough context to size capacity without hidden queues.""" + + combined = "\n".join( + ( + _text("README.md"), + _text("ARCHITECTURE.md"), + _text("docs/operations/deploy-runbook.md"), + _text("CHANGELOG.md"), + ) + ) + + for expected in ( + "NEWSDOM_MAX_CONCURRENT_PARSES", + "429 Too Many Requests", + "Retry-After: 1", + "per process", + "replica", + "before the multipart body", + ): + assert expected in combined + + +def test_capacity_doctoring_records_primary_sources_in_apa_seventh_style() -> None: + """The decision record should preserve standards and security rationale.""" + + doctoring = _text("docs/doctoring/bounded-parser-admission.md") + + for expected in ( + "Nottingham, M., & Fielding, R. (2012)", + "Fielding, R., Nottingham, M., & Reschke, J. (2022)", + "OWASP Foundation. (2023)", + "Python Software Foundation. (2026)", + "RFC 6585", + "RFC 9111", + "API4:2023", + "BoundedSemaphore", + "Rollback", + ): + assert expected in doctoring diff --git a/tests/test_security_policy.py b/tests/test_security_policy.py index 401b8b95..3cabd63e 100644 --- a/tests/test_security_policy.py +++ b/tests/test_security_policy.py @@ -14,8 +14,12 @@ def test_security_policy_covers_reporting_and_supported_branches(): def test_security_policy_includes_explicit_reporting_links(): text = Path("SECURITY.md").read_text(encoding="utf-8") - assert "https://github.com/Seongho-Bae/newsdom-api/security/advisories/new" in text - assert "https://github.com/seonghobae" in text + assert ( + "https://github.com/ContextualWisdomLab/newsdom-api/security/advisories/new" + in text + ) + assert "ContextualWisdomLab repository maintainers" in text + assert "Do not open a public issue, pull-request comment, or discussion" in text def test_readme_and_contributing_link_security_policy(): diff --git a/tests/test_upload_ingestion_benchmark.py b/tests/test_upload_ingestion_benchmark.py new file mode 100644 index 00000000..1be8ddc5 --- /dev/null +++ b/tests/test_upload_ingestion_benchmark.py @@ -0,0 +1,392 @@ +"""Contracts for the upload-ingestion benchmark and raw evidence schema.""" + +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +import pytest + +from tools import benchmark_upload_ingestion as benchmark + + +def _environment_manifest() -> dict[str, int | str]: + """Return complete host evidence for a deterministic benchmark report.""" + + return { + "execution_image": "host:ubuntu-24.04", + "cpu_model": "Test CPU", + "memory_bytes": 8 * 1024**3, + "storage_device": "test-volume", + "filesystem": "ext4", + "dependency_lock_sha256": "sha256:" + "a" * 64, + "commit_sha": "b" * 40, + "worker_count": 1, + "cache_state": "cold", + } + + +def _sample( + duration: float = 0.25, + *, + read_calls: int = 3, + bytes_copied: int = 100, +) -> dict[str, int | float]: + """Return one compact request sample for aggregate-metric tests.""" + + return { + "duration_seconds": duration, + "read_calls": read_calls, + "bytes_copied": bytes_copied, + } + + +def test_candidate_matrix_preserves_every_issue_534_chunk_option() -> None: + """The harness must not silently omit or preselect one requested candidate.""" + + assert benchmark.CHUNK_CANDIDATES == { + "8kib": 8 * 1024, + "64kib": 64 * 1024, + "256kib": 256 * 1024, + "1mib": 1024 * 1024, + "adaptive": None, + } + assert benchmark.CONCURRENCY_LEVELS == (1, 8, 32, 128) + assert benchmark.TARGET_FIXTURE_BYTES == ( + 1 * 1024 * 1024, + 5 * 1024 * 1024, + 20 * 1024 * 1024, + ) + + +@pytest.mark.parametrize( + ("fixture_bytes", "expected"), + [ + (1 * 1024 * 1024, 64 * 1024), + (5 * 1024 * 1024, 256 * 1024), + (20 * 1024 * 1024, 1024 * 1024), + ], +) +def test_adaptive_candidate_resolves_deterministically( + fixture_bytes: int, + expected: int, +) -> None: + """The adaptive candidate needs an auditable size rule before measurement.""" + + assert benchmark.resolve_chunk_bytes("adaptive", fixture_bytes) == expected + + +def test_unknown_candidate_is_rejected() -> None: + """A typo must not fall back to an unrecorded upload policy.""" + + with pytest.raises(ValueError, match="Unknown chunk candidate"): + benchmark.resolve_chunk_bytes("unexpected", 1024) + + +def test_fixture_inventory_rejects_non_pdf_magic(tmp_path: Path) -> None: + """Raw benchmark evidence must identify actual PDF inputs.""" + + fixture = tmp_path / "invalid.pdf" + fixture.write_bytes(b"not a PDF") + + with pytest.raises(ValueError, match="PDF magic"): + benchmark.inventory_fixture(fixture) + + +@pytest.mark.parametrize( + ("size_bytes", "accepted"), + [ + (20 * 1024 * 1024, True), + (20 * 1024 * 1024 + 1, False), + ], +) +def test_fixture_inventory_enforces_twenty_mib_boundary( + tmp_path: Path, + size_bytes: int, + accepted: bool, +) -> None: + """The fixture contract accepts the boundary and rejects oversized input.""" + + fixture = tmp_path / "boundary.pdf" + with fixture.open("wb") as fixture_handle: + fixture_handle.write(b"%PDF-") + fixture_handle.seek(size_bytes - 1) + fixture_handle.write(b"\0") + + if accepted: + assert benchmark.inventory_fixture(fixture)["size_bytes"] == size_bytes + else: + with pytest.raises(ValueError, match="20 MiB"): + benchmark.inventory_fixture(fixture) + + +def test_fixture_inventory_records_hash_and_size(tmp_path: Path) -> None: + """Every benchmark case should be traceable to immutable fixture bytes.""" + + fixture = tmp_path / "fixture.pdf" + fixture.write_bytes(b"%PDF-1.4\nsynthetic benchmark bytes\n%%EOF\n") + + record = benchmark.inventory_fixture(fixture) + + assert record["fixture_name"] == "fixture.pdf" + assert record["size_bytes"] == fixture.stat().st_size + assert record["sha256"].startswith("sha256:") + assert len(record["sha256"]) == len("sha256:") + 64 + + +def test_percentile_uses_nearest_rank_for_small_reproducible_samples() -> None: + """The raw report should define deterministic p50 and p95 aggregation.""" + + values = [0.4, 0.1, 0.3, 0.2] + + assert benchmark.nearest_rank_percentile(values, 0.50) == 0.2 + assert benchmark.nearest_rank_percentile(values, 0.95) == 0.4 + + +def test_current_rss_reads_linux_vmrss_in_bytes(tmp_path: Path) -> None: + """Per-case memory evidence must use current RSS rather than lifetime high-water.""" + + status_path = tmp_path / "status" + status_path.write_text("Name:\tpython\nVmRSS:\t1234 kB\n", encoding="utf-8") + + assert benchmark.current_rss_bytes(status_path=status_path) == 1234 * 1024 + + +def test_current_rss_returns_zero_when_status_is_missing_or_malformed( + tmp_path: Path, +) -> None: + """Unsupported memory providers should be explicit zero evidence, not stale RSS.""" + + missing = tmp_path / "missing" + malformed = tmp_path / "status" + malformed.write_text("VmRSS:\tnot-a-number kB\n", encoding="utf-8") + + assert benchmark.current_rss_bytes(status_path=missing) == 0 + assert benchmark.current_rss_bytes(status_path=malformed) == 0 + + +def test_metrics_use_case_local_rss_and_nearest_rank_event_loop_p95() -> None: + """Memory and responsiveness aggregates must derive from raw cohort evidence.""" + + first = benchmark.CohortObservation( + samples=(_sample(0.10), _sample(0.30)), + wall_seconds=0.40, + cpu_seconds=0.20, + rss_baseline_bytes=100, + peak_rss_bytes=500, + event_loop_delay_samples_seconds=(0.001, 0.002, 0.003, 0.100), + ) + second = benchmark.CohortObservation( + samples=(_sample(0.20),), + wall_seconds=0.20, + cpu_seconds=0.10, + rss_baseline_bytes=150, + peak_rss_bytes=250, + event_loop_delay_samples_seconds=(0.004, 0.005), + ) + + metrics = benchmark.aggregate_metrics((first, second)) + + assert metrics["peak_rss_bytes"] == 500 + assert metrics["peak_rss_delta_bytes"] == 400 + assert metrics["event_loop_p95_delay_seconds"] == 0.100 + assert metrics["event_loop_max_delay_seconds"] == 0.100 + assert metrics["p50_latency_seconds"] == 0.20 + assert metrics["p95_latency_seconds"] == 0.30 + + +def test_later_case_can_report_lower_rss_than_an_earlier_case() -> None: + """A process-lifetime high-water mark must not contaminate later matrix cases.""" + + high = benchmark.CohortObservation( + samples=(_sample(),), + wall_seconds=0.25, + cpu_seconds=0.10, + rss_baseline_bytes=1_000, + peak_rss_bytes=9_000, + event_loop_delay_samples_seconds=(), + ) + low = benchmark.CohortObservation( + samples=(_sample(),), + wall_seconds=0.25, + cpu_seconds=0.10, + rss_baseline_bytes=1_000, + peak_rss_bytes=2_000, + event_loop_delay_samples_seconds=(), + ) + + assert benchmark.aggregate_metrics((high,))["peak_rss_bytes"] == 9_000 + assert benchmark.aggregate_metrics((low,))["peak_rss_bytes"] == 2_000 + + +@pytest.mark.asyncio +async def test_matrix_report_records_environment_cases_and_raw_samples( + tmp_path: Path, +) -> None: + """A small injected run should preserve all inputs and unaggregated samples.""" + + fixture = tmp_path / "fixture.pdf" + fixture.write_bytes(b"%PDF-1.4\nsynthetic benchmark bytes\n%%EOF\n") + output = tmp_path / "report.json" + calls: list[tuple[str, int, int]] = [] + + async def fake_cohort( + fixture_path: Path, + *, + chunk_bytes: int, + concurrency: int, + ) -> benchmark.CohortObservation: + calls.append((fixture_path.name, chunk_bytes, concurrency)) + return benchmark.CohortObservation( + samples=tuple( + { + "duration_seconds": 0.25, + "read_calls": 3, + "bytes_copied": fixture_path.stat().st_size, + } + for _ in range(concurrency) + ), + wall_seconds=0.50, + cpu_seconds=0.25, + rss_baseline_bytes=1_000, + peak_rss_bytes=2_000, + event_loop_delay_samples_seconds=(0.001, 0.010), + ) + + report = await benchmark.run_matrix( + [fixture], + output_path=output, + candidates=("8kib", "adaptive"), + concurrency_levels=(1, 2), + repetitions=2, + environment_manifest=_environment_manifest(), + cohort_runner=fake_cohort, + ) + + assert output.read_text(encoding="utf-8") == json.dumps( + report, + ensure_ascii=False, + indent=2, + sort_keys=True, + ) + "\n" + assert report["schema_version"] == "1.2.0" + assert report["benchmark_environment"]["python_version"] + for field_name, field_value in _environment_manifest().items(): + assert report["benchmark_environment"][field_name] == field_value + assert len(report["fixtures"]) == 1 + assert len(report["cases"]) == 4 + assert len(report["cases"][0]["samples"]) == 2 + assert len(report["cases"][0]["cohort_observations"]) == 2 + assert report["cases"][0]["metrics"]["p50_latency_seconds"] == 0.25 + assert report["cases"][0]["metrics"]["p95_latency_seconds"] == 0.25 + assert report["cases"][0]["metrics"]["peak_rss_delta_bytes"] == 1_000 + assert report["cases"][0]["metrics"]["event_loop_p95_delay_seconds"] == 0.010 + assert calls == [ + ("fixture.pdf", 8 * 1024, 1), + ("fixture.pdf", 8 * 1024, 1), + ("fixture.pdf", 64 * 1024, 1), + ("fixture.pdf", 64 * 1024, 1), + ("fixture.pdf", 8 * 1024, 2), + ("fixture.pdf", 8 * 1024, 2), + ("fixture.pdf", 64 * 1024, 2), + ("fixture.pdf", 64 * 1024, 2), + ] + + +def test_raw_evidence_schema_is_strict_and_covers_required_metrics() -> None: + """Checked-in evidence must be machine-auditable before results are accepted.""" + + schema_path = ( + Path(__file__).resolve().parents[1] + / "docs" + / "benchmarks" + / "upload-ingestion-result.schema.json" + ) + schema = json.loads(schema_path.read_text(encoding="utf-8")) + case_schema = schema["$defs"]["benchmark_case"] + metric_properties = case_schema["properties"]["metrics"]["properties"] + environment_schema = schema["$defs"]["benchmark_environment"] + required_environment_fields = set(_environment_manifest()) + + assert schema["$schema"] == "https://json-schema.org/draft/2020-12/schema" + assert schema["properties"]["schema_version"] == {"const": "1.2.0"} + assert schema["additionalProperties"] is False + assert case_schema["additionalProperties"] is False + assert environment_schema["additionalProperties"] is False + assert required_environment_fields <= set(environment_schema["required"]) + assert required_environment_fields <= set(environment_schema["properties"]) + assert "cohort_observations" in case_schema["required"] + assert case_schema["properties"]["concurrency"] == { + "type": "integer", + "minimum": 1, + } + for required_metric in ( + "p50_latency_seconds", + "p95_latency_seconds", + "throughput_bytes_per_second", + "read_calls", + "cpu_seconds", + "peak_rss_bytes", + "peak_rss_delta_bytes", + "temporary_disk_bytes", + "event_loop_p95_delay_seconds", + "event_loop_max_delay_seconds", + ): + assert required_metric in metric_properties + assert required_metric in case_schema["properties"]["metrics"]["required"] + + +def test_cli_defaults_to_the_complete_evidence_matrix(monkeypatch, tmp_path: Path) -> None: + """The documented command should run every required candidate and concurrency.""" + + fixtures = tmp_path / "fixtures" + fixtures.mkdir() + output = tmp_path / "results.json" + manifest_path = tmp_path / "environment.json" + manifest_path.write_text(json.dumps(_environment_manifest()), encoding="utf-8") + captured: dict[str, object] = {} + + async def fake_run_matrix( + fixture_paths: list[Path], + *, + output_path: Path, + candidates: tuple[str, ...], + concurrency_levels: tuple[int, ...], + repetitions: int, + environment_manifest: dict[str, int | str], + cohort_runner=benchmark.run_cohort, + ) -> dict[str, object]: + captured.update( + fixture_paths=fixture_paths, + output_path=output_path, + candidates=candidates, + concurrency_levels=concurrency_levels, + repetitions=repetitions, + environment_manifest=environment_manifest, + cohort_runner=cohort_runner, + ) + return {} + + monkeypatch.setattr(benchmark, "run_matrix", fake_run_matrix) + monkeypatch.setattr(benchmark, "discover_fixtures", lambda _directory: []) + + benchmark.main( + [ + "--fixtures-dir", + str(fixtures), + "--output", + str(output), + "--environment-manifest", + str(manifest_path), + "--repetitions", + "3", + ] + ) + + assert captured["candidates"] == tuple(benchmark.CHUNK_CANDIDATES) + assert captured["concurrency_levels"] == benchmark.CONCURRENCY_LEVELS + assert captured["repetitions"] == 3 + assert captured["output_path"] == output + assert captured["environment_manifest"] == _environment_manifest() + assert asyncio.iscoroutinefunction(fake_run_matrix) diff --git a/tools/benchmark_upload_ingestion.py b/tools/benchmark_upload_ingestion.py new file mode 100644 index 00000000..8fd6cfa6 --- /dev/null +++ b/tools/benchmark_upload_ingestion.py @@ -0,0 +1,549 @@ +"""Benchmark bounded PDF upload ingestion without selecting a new default.""" + +from __future__ import annotations + +import argparse +import asyncio +import hashlib +import json +import math +import os +import platform +import re +import sys +import tempfile +import time +from collections.abc import Awaitable, Callable, Sequence +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from starlette.datastructures import UploadFile + +CHUNK_CANDIDATES: dict[str, int | None] = { + "8kib": 8 * 1024, + "64kib": 64 * 1024, + "256kib": 256 * 1024, + "1mib": 1024 * 1024, + "adaptive": None, +} +CONCURRENCY_LEVELS = (1, 8, 32, 128) +TARGET_FIXTURE_BYTES = ( + 1 * 1024 * 1024, + 5 * 1024 * 1024, + 20 * 1024 * 1024, +) +_RESULT_SCHEMA_PATH = "docs/benchmarks/upload-ingestion-result.schema.json" +MAX_FIXTURE_BYTES = 20 * 1024 * 1024 +_ENVIRONMENT_MANIFEST_FIELDS = frozenset( + { + "execution_image", + "cpu_model", + "memory_bytes", + "storage_device", + "filesystem", + "dependency_lock_sha256", + "commit_sha", + "worker_count", + "cache_state", + } +) +_SHA256_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$") +_COMMIT_SHA_PATTERN = re.compile(r"^[0-9a-f]{40}$") +SampleRecord = dict[str, int | float] + + +@dataclass(frozen=True, slots=True) +class CohortObservation: + """Raw process and request evidence for one concurrent ingestion cohort.""" + + samples: tuple[SampleRecord, ...] + wall_seconds: float + cpu_seconds: float + rss_baseline_bytes: int + peak_rss_bytes: int + event_loop_delay_samples_seconds: tuple[float, ...] + + def as_record(self) -> dict[str, Any]: + """Return the JSON-serializable cohort evidence retained in the report.""" + + return { + "wall_seconds": self.wall_seconds, + "cpu_seconds": self.cpu_seconds, + "rss_baseline_bytes": self.rss_baseline_bytes, + "peak_rss_bytes": self.peak_rss_bytes, + "peak_rss_delta_bytes": max( + 0, + self.peak_rss_bytes - self.rss_baseline_bytes, + ), + "event_loop_delay_samples_seconds": list( + self.event_loop_delay_samples_seconds + ), + } + + +CohortRunner = Callable[..., Awaitable[CohortObservation]] + + +def resolve_chunk_bytes(candidate: str, fixture_bytes: int) -> int: + """Resolve one named candidate to the exact read size used for a fixture.""" + + if candidate not in CHUNK_CANDIDATES: + raise ValueError(f"Unknown chunk candidate: {candidate}") + configured = CHUNK_CANDIDATES[candidate] + if configured is not None: + return configured + if fixture_bytes <= TARGET_FIXTURE_BYTES[0]: + return 64 * 1024 + if fixture_bytes <= TARGET_FIXTURE_BYTES[1]: + return 256 * 1024 + return 1024 * 1024 + + +def _sha256(path: Path) -> str: + """Return a streaming SHA-256 identifier for one fixture.""" + + digest = hashlib.sha256() + with path.open("rb") as handle: + while chunk := handle.read(1024 * 1024): + digest.update(chunk) + return f"sha256:{digest.hexdigest()}" + + +def inventory_fixture(path: Path) -> dict[str, int | str]: + """Validate and inventory one immutable PDF benchmark fixture.""" + + if not path.is_file() or path.is_symlink(): + raise ValueError(f"Fixture must be a regular non-symlink file: {path}") + size_bytes = path.stat().st_size + if size_bytes > MAX_FIXTURE_BYTES: + raise ValueError(f"Fixture exceeds 20 MiB: {path.name}") + with path.open("rb") as handle: + if handle.read(5) != b"%PDF-": + raise ValueError(f"Fixture does not contain PDF magic bytes: {path.name}") + return { + "fixture_name": path.name, + "size_bytes": size_bytes, + "sha256": _sha256(path), + } + + +def discover_fixtures(directory: Path) -> list[Path]: + """Return the sorted regular PDF fixtures directly under a directory.""" + + if not directory.is_dir(): + raise ValueError(f"Fixture directory does not exist: {directory}") + return sorted( + path + for path in directory.glob("*.pdf") + if path.is_file() and not path.is_symlink() + ) + + +def nearest_rank_percentile(values: Sequence[float], probability: float) -> float: + """Return the deterministic nearest-rank percentile for non-empty samples.""" + + if not values: + raise ValueError("Percentile requires at least one sample") + if not 0 < probability <= 1: + raise ValueError("Percentile probability must be in (0, 1]") + ordered = sorted(values) + rank = max(1, math.ceil(probability * len(ordered))) + return ordered[rank - 1] + + +def current_rss_bytes(*, status_path: Path = Path("/proc/self/status")) -> int: + """Return current Linux process RSS in bytes, or zero when unavailable.""" + + try: + lines = status_path.read_text(encoding="utf-8").splitlines() + except OSError: + return 0 + for line in lines: + if not line.startswith("VmRSS:"): + continue + fields = line.split() + if len(fields) != 3 or fields[2] != "kB": + return 0 + try: + kibibytes = int(fields[1]) + except ValueError: + return 0 + return max(0, kibibytes) * 1024 + return 0 + + +async def _event_loop_probe( + stop_event: asyncio.Event, + observed_delays: list[float], + *, + interval_seconds: float = 0.005, +) -> None: + """Sample event-loop scheduling delays while one cohort is active.""" + + expected = time.perf_counter() + interval_seconds + while not stop_event.is_set(): + await asyncio.sleep(interval_seconds) + now = time.perf_counter() + observed_delays.append(max(0.0, now - expected)) + expected = now + interval_seconds + + +async def _rss_probe( + stop_event: asyncio.Event, + observed_rss_bytes: list[int], + *, + rss_reader: Callable[[], int] = current_rss_bytes, + interval_seconds: float = 0.005, +) -> None: + """Sample current process RSS so each case has an independent peak.""" + + observed_rss_bytes.append(rss_reader()) + while not stop_event.is_set(): + await asyncio.sleep(interval_seconds) + observed_rss_bytes.append(rss_reader()) + + +async def _copy_one_fixture( + fixture_path: Path, + *, + chunk_bytes: int, + temporary_directory: Path, +) -> SampleRecord: + """Copy one real file-backed UploadFile to temporary storage and measure it.""" + + started = time.perf_counter() + read_calls = 0 + bytes_copied = 0 + source_handle = fixture_path.open("rb") + upload = UploadFile( + file=source_handle, + size=fixture_path.stat().st_size, + filename=fixture_path.name, + ) + try: + with tempfile.NamedTemporaryFile( + dir=temporary_directory, + prefix="newsdom-ingestion-benchmark-", + delete=True, + ) as destination: + while True: + chunk = await upload.read(chunk_bytes) + read_calls += 1 + if not chunk: + break + destination.write(chunk) + bytes_copied += len(chunk) + destination.flush() + finally: + await upload.close() + return { + "duration_seconds": time.perf_counter() - started, + "read_calls": read_calls, + "bytes_copied": bytes_copied, + } + + +async def run_cohort( + fixture_path: Path, + *, + chunk_bytes: int, + concurrency: int, +) -> CohortObservation: + """Run one real-file cohort and retain raw timing, loop, and RSS evidence.""" + + if concurrency < 1: + raise ValueError("Concurrency must be at least one") + if chunk_bytes < 1: + raise ValueError("Chunk size must be at least one byte") + + wall_started = time.perf_counter() + cpu_started = time.process_time() + rss_baseline_bytes = current_rss_bytes() + observed_rss_bytes = [rss_baseline_bytes] + stop_event = asyncio.Event() + observed_delays: list[float] = [] + with tempfile.TemporaryDirectory(prefix="newsdom-ingestion-benchmark-") as root: + temporary_directory = Path(root) + event_loop_probe = asyncio.create_task( + _event_loop_probe(stop_event, observed_delays) + ) + rss_probe = asyncio.create_task(_rss_probe(stop_event, observed_rss_bytes)) + try: + samples = await asyncio.gather( + *( + _copy_one_fixture( + fixture_path, + chunk_bytes=chunk_bytes, + temporary_directory=temporary_directory, + ) + for _ in range(concurrency) + ) + ) + finally: + observed_rss_bytes.append(current_rss_bytes()) + stop_event.set() + await asyncio.gather(event_loop_probe, rss_probe) + + return CohortObservation( + samples=tuple(samples), + wall_seconds=time.perf_counter() - wall_started, + cpu_seconds=time.process_time() - cpu_started, + rss_baseline_bytes=rss_baseline_bytes, + peak_rss_bytes=max(observed_rss_bytes, default=0), + event_loop_delay_samples_seconds=tuple(observed_delays), + ) + + +def _environment_record() -> dict[str, Any]: + """Return reproducibility metadata that does not contain credentials.""" + + return { + "python_version": platform.python_version(), + "python_implementation": platform.python_implementation(), + "executable": Path(sys.executable).name, + "platform": platform.platform(), + "machine": platform.machine(), + "processor_count": os.cpu_count(), + "rss_provider": "linux_proc_status_vmrss", + } + + +def _validated_environment_manifest( + manifest: dict[str, Any], +) -> dict[str, int | str]: + """Validate caller-supplied hardware and execution provenance.""" + + field_names = set(manifest) + if field_names != _ENVIRONMENT_MANIFEST_FIELDS: + missing = sorted(_ENVIRONMENT_MANIFEST_FIELDS - field_names) + unexpected = sorted(field_names - _ENVIRONMENT_MANIFEST_FIELDS) + raise ValueError( + f"Environment manifest fields mismatch; missing={missing}, " + f"unexpected={unexpected}" + ) + string_fields = ( + "execution_image", + "cpu_model", + "storage_device", + "filesystem", + ) + if any( + not isinstance(manifest[field_name], str) + or not str(manifest[field_name]).strip() + for field_name in string_fields + ): + raise ValueError("Environment manifest string fields must be non-empty") + if ( + not isinstance(manifest["memory_bytes"], int) + or manifest["memory_bytes"] < 1 + or not isinstance(manifest["worker_count"], int) + or manifest["worker_count"] < 1 + ): + raise ValueError("Environment manifest counts must be positive integers") + dependency_lock = manifest["dependency_lock_sha256"] + commit_sha = manifest["commit_sha"] + if ( + not isinstance(dependency_lock, str) + or _SHA256_PATTERN.fullmatch(dependency_lock) is None + or not isinstance(commit_sha, str) + or _COMMIT_SHA_PATTERN.fullmatch(commit_sha) is None + ): + raise ValueError("Environment manifest hashes are invalid") + if manifest["cache_state"] not in {"cold", "warm", "disabled"}: + raise ValueError("Environment manifest cache_state is invalid") + return dict(manifest) + + +def aggregate_metrics( + cohorts: Sequence[CohortObservation], +) -> dict[str, int | float]: + """Aggregate request and cohort evidence without lifetime RSS contamination.""" + + samples = [sample for cohort in cohorts for sample in cohort.samples] + if not samples: + raise ValueError("At least one request sample is required") + durations = [float(sample["duration_seconds"]) for sample in samples] + total_bytes = sum(int(sample["bytes_copied"]) for sample in samples) + total_wall = sum(cohort.wall_seconds for cohort in cohorts) + loop_delays = [ + delay + for cohort in cohorts + for delay in cohort.event_loop_delay_samples_seconds + ] + return { + "p50_latency_seconds": nearest_rank_percentile(durations, 0.50), + "p95_latency_seconds": nearest_rank_percentile(durations, 0.95), + "throughput_bytes_per_second": total_bytes / total_wall if total_wall else 0.0, + "read_calls": sum(int(sample["read_calls"]) for sample in samples), + "cpu_seconds": sum(cohort.cpu_seconds for cohort in cohorts), + "peak_rss_bytes": max( + (cohort.peak_rss_bytes for cohort in cohorts), + default=0, + ), + "peak_rss_delta_bytes": max( + ( + max(0, cohort.peak_rss_bytes - cohort.rss_baseline_bytes) + for cohort in cohorts + ), + default=0, + ), + "temporary_disk_bytes": total_bytes, + "event_loop_p95_delay_seconds": ( + nearest_rank_percentile(loop_delays, 0.95) if loop_delays else 0.0 + ), + "event_loop_max_delay_seconds": max(loop_delays, default=0.0), + } + + +async def run_matrix( + fixture_paths: list[Path], + *, + output_path: Path, + candidates: tuple[str, ...] = tuple(CHUNK_CANDIDATES), + concurrency_levels: tuple[int, ...] = CONCURRENCY_LEVELS, + repetitions: int = 3, + environment_manifest: dict[str, Any], + cohort_runner: CohortRunner = run_cohort, +) -> dict[str, Any]: + """Run and write the reproducible raw upload-ingestion evidence matrix.""" + + if repetitions < 1: + raise ValueError("Repetitions must be at least one") + if not fixture_paths: + raise ValueError("At least one PDF fixture is required") + + benchmark_environment = _environment_record() + benchmark_environment.update(_validated_environment_manifest(environment_manifest)) + fixture_records = [inventory_fixture(path) for path in fixture_paths] + cases: list[dict[str, Any]] = [] + for concurrency in concurrency_levels: + if concurrency < 1: + raise ValueError("Concurrency levels must be positive") + for fixture_path, fixture_record in zip(fixture_paths, fixture_records): + fixture_bytes = int(fixture_record["size_bytes"]) + for candidate in candidates: + chunk_bytes = resolve_chunk_bytes(candidate, fixture_bytes) + cohorts = [ + await cohort_runner( + fixture_path, + chunk_bytes=chunk_bytes, + concurrency=concurrency, + ) + for _ in range(repetitions) + ] + all_samples = [ + sample + for cohort in cohorts + for sample in cohort.samples + ] + cases.append( + { + "fixture_name": fixture_path.name, + "candidate": candidate, + "chunk_bytes": chunk_bytes, + "concurrency": concurrency, + "repetitions": repetitions, + "metrics": aggregate_metrics(cohorts), + "cohort_observations": [ + cohort.as_record() for cohort in cohorts + ], + "samples": all_samples, + } + ) + + report: dict[str, Any] = { + "schema_version": "1.2.0", + "schema_path": _RESULT_SCHEMA_PATH, + "generated_at": datetime.now(timezone.utc).isoformat(), + "benchmark_environment": benchmark_environment, + "fixtures": fixture_records, + "cases": cases, + } + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text( + json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return report + + +def _positive_int(value: str) -> int: + """Parse one strictly positive command-line integer.""" + + parsed = int(value) + if parsed < 1: + raise argparse.ArgumentTypeError("value must be at least one") + return parsed + + +def main(argv: list[str] | None = None) -> None: + """Run the complete issue #534 candidate matrix from the command line.""" + + parser = argparse.ArgumentParser( + description=( + "Benchmark PDF UploadFile ingestion candidates and emit raw JSON evidence." + ) + ) + parser.add_argument( + "--fixtures-dir", + type=Path, + required=True, + help="Directory containing real 1 MiB, 5 MiB, and 20 MiB PDF fixtures.", + ) + parser.add_argument( + "--output", + type=Path, + required=True, + help="Path for the schema-versioned raw JSON report.", + ) + parser.add_argument( + "--environment-manifest", + type=Path, + required=True, + help="Strict JSON hardware, lock, revision, worker, and cache evidence.", + ) + parser.add_argument( + "--repetitions", + type=_positive_int, + default=3, + help="Cohort repetitions for every matrix case (default: 3).", + ) + parser.add_argument( + "--candidate", + dest="candidates", + action="append", + choices=tuple(CHUNK_CANDIDATES), + help="Optional candidate filter; repeat to select multiple values.", + ) + parser.add_argument( + "--concurrency", + dest="concurrency_levels", + action="append", + type=_positive_int, + help="Optional concurrency filter; repeat to select multiple values.", + ) + args = parser.parse_args(argv) + fixture_paths = discover_fixtures(args.fixtures_dir) + environment_manifest = json.loads( + args.environment_manifest.read_text(encoding="utf-8") + ) + if not isinstance(environment_manifest, dict): + raise ValueError("Environment manifest must be a JSON object") + candidates = tuple(args.candidates or CHUNK_CANDIDATES) + concurrency_levels = tuple(args.concurrency_levels or CONCURRENCY_LEVELS) + asyncio.run( + run_matrix( + fixture_paths, + output_path=args.output, + candidates=candidates, + concurrency_levels=concurrency_levels, + repetitions=args.repetitions, + environment_manifest=environment_manifest, + ) + ) + + +if __name__ == "__main__": # pragma: no cover + main()