From 25b7c8ebb7f7e1ed4714627a56103f2dbd425d8d Mon Sep 17 00:00:00 2001 From: seonghobae <8172694+seonghobae@users.noreply.github.com> Date: Tue, 4 Aug 2026 21:55:03 +0000 Subject: [PATCH 01/75] =?UTF-8?q?=F0=9F=9B=A1=EF=B8=8F=20Sentinel:=20[HIGH?= =?UTF-8?q?]=20Fix=20DoS=20from=20hmac.compare=5Fdigest=20Unicode=20except?= =?UTF-8?q?ion?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ๐Ÿšจ Severity: HIGH ๐Ÿ’ก Vulnerability: `hmac.compare_digest` throws an unhandled `TypeError` when it receives non-ASCII strings. By sending an `Authorization` header with non-ASCII characters, an unauthenticated user could trigger an unhandled 500 Internal Server Error, leading to a potential Denial of Service (DoS). ๐ŸŽฏ Impact: Attackers could cause application crashes by passing Unicode characters in the Authorization header. ๐Ÿ”ง Fix: Encoded the inputs to bytes via `.encode("utf-8")` before comparing, which `hmac.compare_digest` processes safely. Added test for non-ASCII input handling. โœ… Verification: Ran `uv run pytest` successfully. --- .jules/sentinel.md | 4 ++++ src/newsdom_api/main.py | 2 +- tests/test_auth.py | 14 ++++++++++++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/.jules/sentinel.md b/.jules/sentinel.md index 2b5d819c..9b64a490 100644 --- a/.jules/sentinel.md +++ b/.jules/sentinel.md @@ -90,3 +90,7 @@ **Vulnerability:** The `_safe_upload_filename` function used `filename.replace`, `PurePosixPath`, and `re.sub` on unbounded client input, making it vulnerable to ReDoS or CPU/memory exhaustion (DoS) when fed extremely long strings. **Learning:** Even fast standard library functions like `PurePosixPath` and string replacements can cause significant lag when chained on strings in the megabytes. String processing operations should always bound their inputs first if the input is untrusted and can be arbitrarily large. **Prevention:** Cap the length of client-provided filename strings early by slicing them (e.g. `filename = filename[-512:]`) before doing more complex string parsing or regex replacements, especially when only the basename suffix is relevant. +## 2026-08-04 - hmac.compare_digest Unicode Exception (DoS) +**Vulnerability:** Unhandled TypeError from hmac.compare_digest when receiving non-ASCII input. +**Learning:** Python's hmac.compare_digest throws a TypeError on non-ASCII characters, which could lead to an unhandled 500 error if placed in an auth flow. +**Prevention:** Encode all inputs to bytes using .encode('utf-8') before comparing. diff --git a/src/newsdom_api/main.py b/src/newsdom_api/main.py index 4efdad56..ca5b7599 100644 --- a/src/newsdom_api/main.py +++ b/src/newsdom_api/main.py @@ -133,7 +133,7 @@ def require_authorization( return expected = f"Bearer {token}" provided = authorization or "" - if not hmac.compare_digest(provided, expected): + if not hmac.compare_digest(provided.encode("utf-8"), expected.encode("utf-8")): raise HTTPException( status_code=401, detail=UNAUTHORIZED_DETAIL, diff --git a/tests/test_auth.py b/tests/test_auth.py index 2dc94fa2..88cf6412 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -53,6 +53,20 @@ def test_parse_rejects_invalid_bearer_when_secret_set(monkeypatch, stub_parser): assert response.json()["detail"] == "Unauthorized" +def test_parse_rejects_non_ascii_bearer_when_secret_set(monkeypatch, stub_parser): + monkeypatch.setenv(API_TOKEN_ENV_VAR, "s3cret-token") + client = TestClient(app, raise_server_exceptions=False) + # The ASGI spec and TestClient headers require raw byte tuples to inject non-ASCII + # natively without framework-level pre-validation failures. + response = client.post( + "/parse", + files=_PDF_FILES, + headers=[(b"Authorization", b"Bearer \xe2\x98\x83")], + ) + assert response.status_code == 401 + assert response.json()["detail"] == "Unauthorized" + + def test_parse_accepts_valid_bearer_when_secret_set(monkeypatch, stub_parser): monkeypatch.setenv(API_TOKEN_ENV_VAR, "s3cret-token") client = TestClient(app) From b5bea27b20461f4774f4e67e8f0ade820e4a46e6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 5 Aug 2026 20:52:51 +0900 Subject: [PATCH 02/75] test(auth): require fail-closed parser readiness --- tests/test_auth_fail_closed_contract.py | 71 +++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 tests/test_auth_fail_closed_contract.py diff --git a/tests/test_auth_fail_closed_contract.py b/tests/test_auth_fail_closed_contract.py new file mode 100644 index 00000000..7a1d9520 --- /dev/null +++ b/tests/test_auth_fail_closed_contract.py @@ -0,0 +1,71 @@ +"""Fail-closed authentication and readiness regression contracts.""" + +from __future__ import annotations + +from fastapi.testclient import TestClient + +from newsdom_api.config import API_TOKEN_ENV_VAR +from newsdom_api.main import app + +_PDF_FILES = { + "file": ("fixture.pdf", b"%PDF-1.4\n%synthetic\n", "application/pdf") +} + + +def test_default_configuration_without_token_blocks_parser_before_work( + monkeypatch, +) -> None: + """Required authentication must not silently become open when its token is absent.""" + + monkeypatch.delenv(API_TOKEN_ENV_VAR, raising=False) + monkeypatch.delenv("NEWSDOM_AUTH_MODE", raising=False) + monkeypatch.delenv("NEWSDOM_RUNTIME_PROFILE", raising=False) + parser_called = False + + def fake_parse_pdf(*_args, **_kwargs): + nonlocal parser_called + parser_called = True + return {"document_id": "fixture", "pages": []} + + monkeypatch.setattr("newsdom_api.main._validate_pdf_structure", lambda _: None) + monkeypatch.setattr("newsdom_api.main.parse_pdf", fake_parse_pdf) + + response = TestClient(app, raise_server_exceptions=False).post( + "/parse", files=_PDF_FILES + ) + + assert response.status_code == 503 + assert response.json() == {"detail": "Service Unavailable"} + assert parser_called is False + + +def test_ready_fails_closed_when_required_authentication_is_unconfigured( + monkeypatch, +) -> None: + """Traffic readiness must expose invalid auth configuration without secret detail.""" + + monkeypatch.delenv(API_TOKEN_ENV_VAR, raising=False) + monkeypatch.delenv("NEWSDOM_AUTH_MODE", raising=False) + monkeypatch.delenv("NEWSDOM_RUNTIME_PROFILE", raising=False) + + response = TestClient(app, raise_server_exceptions=False).get("/ready") + + assert response.status_code == 503 + assert response.json() == {"detail": "Service Unavailable"} + assert "token" not in response.text.lower() + assert "environment" not in response.text.lower() + + +def test_health_remains_liveness_only_when_authentication_is_unconfigured( + monkeypatch, +) -> None: + """Liveness must remain independent from authentication readiness.""" + + monkeypatch.delenv(API_TOKEN_ENV_VAR, raising=False) + monkeypatch.delenv("NEWSDOM_AUTH_MODE", raising=False) + monkeypatch.delenv("NEWSDOM_RUNTIME_PROFILE", raising=False) + + response = TestClient(app, raise_server_exceptions=False).get("/health") + + assert response.status_code == 200 + assert response.json() == {"status": "ok"} From 6ec3e857fd3a04e6259f30cb6306ea78967990c6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 5 Aug 2026 21:54:23 +0900 Subject: [PATCH 03/75] feat(auth): freeze fail-closed runtime and readiness --- .env.example | 7 + CHANGELOG.md | 11 +- README.md | 54 ++- docker-compose.yml | 19 +- .../fail-closed-parser-authentication.md | 112 +++++ docs/operations/deploy-runbook.md | 30 ++ docs/operations/kubernetes-deployment.yaml | 65 +++ src/newsdom_api/config.py | 137 +++++- src/newsdom_api/main.py | 306 ++++++++----- src/newsdom_api/mineru_runner.py | 13 + src/newsdom_api/schemas.py | 18 +- tests/conftest.py | 7 + tests/test_auth.py | 430 +++++++++++++++--- tests/test_auth_deployment_contract.py | 111 +++++ tests/test_auth_fail_closed_contract.py | 57 ++- tests/test_health.py | 2 +- tests/test_runtime_readiness.py | 73 +++ 17 files changed, 1214 insertions(+), 238 deletions(-) create mode 100644 .env.example create mode 100644 docs/doctoring/fail-closed-parser-authentication.md create mode 100644 docs/operations/kubernetes-deployment.yaml create mode 100644 tests/test_auth_deployment_contract.py create mode 100644 tests/test_runtime_readiness.py diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..e18ae5f8 --- /dev/null +++ b/.env.example @@ -0,0 +1,7 @@ +# Production and shared environments use the fail-closed authentication mode. +NEWSDOM_AUTH_MODE=required +NEWSDOM_RUNTIME_PROFILE=production +# Generate a high-entropy token in the deployment secret store. Never commit it. +NEWSDOM_API_TOKEN= +# Optional API-only image override for a bundled or mounted MinerU executable. +# NEWSDOM_MINERU_BIN=/opt/mineru/bin/mineru diff --git a/CHANGELOG.md b/CHANGELOG.md index 437c75fe..64776855 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,12 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +> **0.3.0 deployment migration:** parser authentication changes from +> **default-open** to **default-required**. Production must configure +> `NEWSDOM_AUTH_MODE=required`, `NEWSDOM_RUNTIME_PROFILE=production`, and +> `NEWSDOM_API_TOKEN`; the explicit disabled mode is development-only. + + ### Changed - `/parse`๋ฅผ ์–ธ์–ด ์„ ํƒํ˜• ํŒŒ์„œ๋กœ ์ผ๋ฐ˜ํ™”: MinerU `-l japan`/`-m ocr` ํ•˜๋“œ์ฝ”๋”ฉ์„ ์ œ๊ฑฐํ•˜๊ณ  optional form ํ•„๋“œ `language`(MinerU 3.4.4 ๊ณต์‹ ๊ธฐ๋ณธ `ch`, ๊ณต๊ฐœ ์–ธ์–ด๊ตฐ/alias ๊ฒ€์ฆ)์™€ `mode`(`auto`/`ocr`/`txt`, ๊ธฐ๋ณธ `auto`)๋กœ ํŒŒ๋ผ๋ฏธํ„ฐํ™”. `mode=auto`๋Š” born-digital PDF๊ฐ€ ๊ฐ•์ œ OCR์„ ๊ฑด๋„ˆ๋›ฐ๋„๋ก ํ•จ. ๊ธฐ์กด ์ž…๋ ฅ `language=japan&mode=ocr`๋Š” ๊ณต์‹ ๊ทœ์•ฝ๋Œ€๋กœ `ch`/`ocr`๋กœ ์ •๊ทœํ™”๋จ. - OpenAPI ์ œ๋ชฉ/์„ค๋ช…, README, `ArticleNode.headline` ๋ฌธ์„œ๋ฅผ ์ผ๋ฐ˜ ๋ฌธ์„œ์šฉ (section heading) ํ‘œํ˜„์œผ๋กœ ์žฌ๊ตฌ์„ฑํ•˜์—ฌ ํŠน์ • ์–ธ์–ด/์‹ ๋ฌธ ๊ฐ€์ •์„ ์†Œ๋น„์ž์—๊ฒŒ ๋…ธ์ถœํ•˜์ง€ ์•Š๋„๋ก ํ•จ. ์‘๋‹ต ์Šคํ‚ค๋งˆ ํ•„๋“œ๋Š” ํ•˜์œ„ ํ˜ธํ™˜์„ ์œ„ํ•ด ๋ณ€๊ฒฝํ•˜์ง€ ์•Š์Œ. ### Added -- [UX/DX] OpenAPI Swagger ๋ฌธ์„œ ์ดํ•ด๋„ ํ–ฅ์ƒ์„ ์œ„ํ•ด `ParseQuality` ๋ฐ `HealthResponse` ๋ชจ๋ธ์— ๊ตฌ์ฒด์ ์ธ `json_schema_extra` ์˜ˆ์ œ ์ถ”๊ฐ€. (Palette) - [CLI] ๋‹จ์ผ NewsDOM JSON ํŒŒ์ผ์„ ํŽ˜์ด์ง€ ๋‹จ์œ„๋กœ ๋ถ„๋ฆฌํ•˜๋Š” `tools/split_dom.py` ๋„๊ตฌ๋ฅผ ์ถ”๊ฐ€ํ–ˆ์Šต๋‹ˆ๋‹ค. - [CLI] NewsDOM JSON ํŒŒ์ผ์˜ ๋ชจ๋“  ํ…์ŠคํŠธ ๋‚ด์šฉ์„ ๋งˆ์Šคํ‚นํ•˜์—ฌ ์ต๋ช…ํ™”ํ•˜๋Š” `tools/anonymize_dom.py` ๋„๊ตฌ๋ฅผ ์ถ”๊ฐ€ํ–ˆ์Šต๋‹ˆ๋‹ค. - `/parse`์— ๋Œ€ํ•œ optional bearer ์ธ์ฆ ๊ฒŒ์ดํŠธ: ๋ฆฌํ”„ ์„œ๋น„์Šค ์ž์ฒด ์„ค์ • `NEWSDOM_API_TOKEN`์ด ์„ค์ •๋˜๋ฉด `Authorization: Bearer `์„ ์š”๊ตฌ(์ƒ์ˆ˜ ์‹œ๊ฐ„ ๋น„๊ต), ๋ฏธ์„ค์ • ์‹œ ๊ฐœ๋ฐฉ(๊ฐœ๋ฐœ์šฉ). `/health`๋Š” ํ•ญ์ƒ ๋ฏธ์ธ์ฆ ์œ ์ง€. @@ -20,11 +25,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [CLI] ํŒŒ์‹ฑ๋œ NewsDOM JSON์—์„œ ์ˆœ์ˆ˜ ํ…์ŠคํŠธ ๋ฐ์ดํ„ฐ๋ฅผ ์ถ”์ถœํ•˜์—ฌ ํ…์ŠคํŠธ ํŒŒ์ผ ๋˜๋Š” stdout์œผ๋กœ ์ถœ๋ ฅํ•˜๋Š” `tools/extract_text.py` ๋„๊ตฌ๋ฅผ ์ถ”๊ฐ€ํ–ˆ์Šต๋‹ˆ๋‹ค. ### Security +- `/parse` authentication is now immutable per application instance and fails closed before multipart body parsing when required configuration is missing. Hostile missing, invalid, Unicode, oversized, and duplicated Authorization headers return one non-sensitive response. +- Added unauthenticated `/ready` traffic readiness that combines authentication configuration with MinerU executable availability while `/health` remains liveness-only. - ์ „์—ญ 500 ์—๋Ÿฌ ์‘๋‹ต์—๋„ ํ‘œ์ค€ ๋ณด์•ˆ ํ—ค๋”๋ฅผ ์ ์šฉํ•˜์—ฌ ์˜ˆ์™ธ ๊ฒฝ๋กœ์—์„œ header ๋ˆ„๋ฝ์„ ๋ฐฉ์ง€ - MinerU subprocess argv ์ƒ์„ฑ ์‹œ `-`๋กœ ์‹œ์ž‘ํ•˜๋Š” option-like ์ธ์ž๋ฅผ ๊ฑฐ๋ถ€ํ•˜์—ฌ argument injection ์œ„ํ—˜์„ ๋‚ฎ์ถค - API ์—๋Ÿฌ ์‘๋‹ต ์ƒ์„ฑ ์‹œ ๋‚ด๋ถ€ ์˜ˆ์™ธ ์ฒด์ธ์„ ์–ต์ œํ•˜์—ฌ ์˜์กด์„ฑ ์˜ค๋ฅ˜๋‚˜ ๋‚ด๋ถ€ ๊ฒฝ๋กœ๊ฐ€ ๋…ธ์ถœ๋  ๊ฐ€๋Šฅ์„ฑ์„ ์ค„์ž„ - API ์‘๋‹ต ๋ฏธ๋“ค์›จ์–ด์— `Cache-Control: no-store, max-age=0` ํ—ค๋”๋ฅผ ์ถ”๊ฐ€ํ•˜์—ฌ ๋ฏผ๊ฐํ•œ ํŒŒ์‹ฑ ๋ฐ์ดํ„ฐ์˜ ๋ธŒ๋ผ์šฐ์ € ๋ฐ ์ค‘๊ฐ„ ์บ์‹ฑ์„ ๋ฐฉ์ง€ -- `uv.lock`์˜ ์˜์กด์„ฑ์„ ์žฌ์ž ๊ธˆํ•˜์—ฌ ์‹ค์ œ `pip-audit`/`trivy-fs` CVE๋ฅผ ์ œ๊ฑฐ: ๋Ÿฐํƒ€์ž„ ๊ฒฝ๋กœ์˜ `pillow` 12.2.0โ†’12.3.0 (PYSEC-2026-3451/3452/3453/3454/3493/3494/3495/3496, ์ด๋ฏธ์ง€ ํŒŒ์„œ ์ทจ์•ฝ์  8๊ฑด), `pypdf` 6.13.3โ†’6.14.2 (CVE-2026-59935/59936/59937/59938, PDF ํŒŒ์‹ฑ ๊ฒฝ๋กœ), `click` 8.3.2โ†’8.4.2 (PYSEC-2026-2132) โ€” ๋ชจ๋‘ ์Šค์บ” PDF/์ด๋ฏธ์ง€ ํŒŒ์‹ฑ ๋Ÿฐํƒ€์ž„์— ์ง์ ‘ ๊ด€๋ จ. ๋นŒ๋“œ ๋„๊ตฌ `setuptools` 81.0.0โ†’83.0.0 (CVE-2026-59890). ๋ฌธ์„œ ํˆด์ฒด์ธ์˜ `pymdown-extensions` 10.21.3โ†’11.0.1 (CVE-2026-61632, MEDIUM)์€ `mkdocs-material` 9.6.x์˜ `pymdown-extensions~=10.2`(`<11`) ์ƒํ•œ ๋•Œ๋ฌธ์— ๋ง‰ํ˜€ ์žˆ์—ˆ์œผ๋ฏ€๋กœ, docs extra ํ•€์„ `mkdocs-material>=9.7,<9.8`๋กœ ์˜ฌ๋ ค(9.7.x๋Š” ๊ธฐ์กด `<11` ์ƒํ•œ์„ ์ œ๊ฑฐ) ํ•ด์†Œํ•จ. `uv run mkdocs build --strict` ํ†ต๊ณผ ํ™•์ธ. ์กฐ์น˜ ํ›„ ์ „์ฒด ์ž ๊ธˆ(๋Ÿฐํƒ€์ž„+extras) `pip-audit`: ์ทจ์•ฝ์  0๊ฑด. `pyproject.toml`์˜ ๋Ÿฐํƒ€์ž„ ํ•˜ํ•œ์„ `Pillow>=12.3`, `pypdf>=6.14.2`๋กœ, build-system ํ•˜ํ•œ์„ `setuptools>=83`์œผ๋กœ ํ•จ๊ป˜ ์˜ฌ๋ ค ํ–ฅํ›„ lock ์žฌ์ƒ์„ฑ ์‹œ ์ทจ์•ฝ ๋ฒ”์œ„๋กœ ํšŒ๊ท€ํ•˜์ง€ ์•Š๋„๋ก ๊ณ ์ •. +- `uv.lock`์˜ ์˜์กด์„ฑ์„ ์žฌ์ž ๊ธˆํ•˜์—ฌ ์‹ค์ œ `pip-audit`/`trivy-fs` CVE๋ฅผ ์ œ๊ฑฐ: ๋Ÿฐํƒ€์ž„ ๊ฒฝ๋กœ์˜ `pillow` 12.2.0โ†’12.3.0 (PYSEC-2026-3451/3452/3453/3454/3493/3494/3495/3496, ์ด๋ฏธ์ง€ ํŒŒ์„œ ์ทจ์•ฝ์  8๊ฑด), `pypdf` 6.13.3โ†’6.14.2 (CVE-2026-59935/59936/59937/59938, PDF ํŒŒ์‹ฑ ๊ฒฝ๋กœ), `click` 8.3.2โ†’8.4.2 (PYSEC-2026-2132) โ€” ๋ชจ๋‘ ์Šค์บ” PDF/์ด๋ฏธ์ง€ ํŒŒ์‹ฑ ๋Ÿฐํƒ€์ž„์— ์ง์ ‘ ๊ด€๋ จ๋˜๋ฉฐ `pyproject.toml` ๊ธฐ์กด ๋ฒ”์œ„ ๋‚ด ํ•ด์„. ๋นŒ๋“œ ๋„๊ตฌ `setuptools` 81.0.0โ†’83.0.0 (CVE-2026-59890). ๋ฌธ์„œ ํˆด์ฒด์ธ์˜ `pymdown-extensions` 10.21.3โ†’11.0.1 (CVE-2026-61632, MEDIUM)์€ `mkdocs-material` 9.6.x์˜ `pymdown-extensions~=10.2`(`<11`) ์ƒํ•œ ๋•Œ๋ฌธ์— ๋ง‰ํ˜€ ์žˆ์—ˆ์œผ๋ฏ€๋กœ, docs extra ํ•€์„ `mkdocs-material>=9.7,<9.8`๋กœ ์˜ฌ๋ ค(9.7.x๋Š” ์ƒํ•œ์„ `>=10.2`๋กœ ์™„ํ™”) ํ•ด์†Œํ•จ. `uv run mkdocs build --strict` ํ†ต๊ณผ ํ™•์ธ. ์กฐ์น˜ ํ›„ ์ „์ฒด ์ž ๊ธˆ(๋Ÿฐํƒ€์ž„+extras) `pip-audit`: ์ทจ์•ฝ์  0๊ฑด. ### Performance - `newsdom_api.dom_builder._html_safe_text` ํ•จ์ˆ˜์— early return๊ณผ ํƒ€์ž… ์ฒดํฌ๋ฅผ ๋„์ž…ํ•˜์—ฌ ๋ถˆํ•„์š”ํ•œ `str()` ์บ์ŠคํŒ…์„ ์ œ๊ฑฐํ•จ์œผ๋กœ์จ ์ฒ˜๋ฆฌ ์†๋„๋ฅผ ๊ฐœ์„ ํ–ˆ์Šต๋‹ˆ๋‹ค. diff --git a/README.md b/README.md index 52239a35..79239a40 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ submodule / sidecar in a larger system. official language families and compatibility aliases such as `japan` remain available - Parsing `mode` defaults to `auto` so born-digital text PDFs skip forced OCR -- Optional bearer auth on `/parse`; unauthenticated `/health` liveness probe +- Fail-closed bearer authentication on `/parse`; separate unauthenticated `/health` liveness and `/ready` traffic-readiness probes ## Quickstart @@ -60,24 +60,24 @@ Silicon hosts running the API service inside Docker. The default image ships the API service only and does not bundle the MinerU runtime. `/parse` requires a compatible MinerU runtime to be available inside the container image or exposed through `NEWSDOM_MINERU_BIN`. -> Readiness caveat: `/health` reports process liveness only. Because the -> default image does not bundle MinerU, a container can report a green -> `/health` while `/parse` still returns `503` until a MinerU runtime is -> reachable. Treat MinerU availability as a separate readiness concern when -> wiring this sidecar into a larger system. +> `GET /health` reports process liveness only. `GET /ready` reports traffic +> readiness and remains unavailable until both fail-closed authentication and +> the MinerU runtime are configured. Orchestrators must route traffic by +> `/ready`, not by `/health`. #### docker compose -A `docker-compose.yml` is provided for standalone/sidecar use. Its healthcheck -targets the unauthenticated `/health` endpoint: +A production-oriented `docker-compose.yml` is provided for standalone/sidecar +use. It requires `NEWSDOM_API_TOKEN` before startup and its healthcheck targets +`/ready`: ```bash docker compose up --build ``` -Uncomment `NEWSDOM_MINERU_BIN` (path to a MinerU executable) and/or -`NEWSDOM_API_TOKEN` (bearer secret) in the compose `environment:` block to make -`/parse` functional and/or protected. +Export `NEWSDOM_API_TOKEN` from a secret store before running Compose. Mount or +bundle MinerU and set `NEWSDOM_MINERU_BIN` when using the API-only image. The +service remains live but not ready until both dependencies are valid. #### Building a MinerU-bundled image @@ -140,21 +140,37 @@ The accepted language contract follows the official MinerU canonicalizes `en`, `japan`, `chinese_cht`, and `latin` to `ch`; this sidecar performs the same normalization before launching the subprocess. -#### Authentication +#### Authentication and readiness -`/parse` is unauthenticated by default (development). Set the sidecar's own -`NEWSDOM_API_TOKEN` environment variable to require a bearer token: +Parser authentication is required by default. This replaces the previous +default-open behavior. Configure the explicit production contract and supply the +secret from the deployment secret store: ```bash -NEWSDOM_API_TOKEN=$(openssl rand -hex 32) \ - uv run uvicorn --app-dir src newsdom_api.main:app +export NEWSDOM_AUTH_MODE=required +export NEWSDOM_RUNTIME_PROFILE=production +export NEWSDOM_API_TOKEN=$(openssl rand -hex 32) +uv run uvicorn --app-dir src newsdom_api.main:app curl -F "file=@sample.pdf" -H "Authorization: Bearer $NEWSDOM_API_TOKEN" \ http://127.0.0.1:8000/parse ``` -Requests without a matching `Authorization: Bearer ` header receive -`401`. `/health` stays unauthenticated so orchestrators can always probe it. -Supply the token from your deployment's secret store rather than committing it. +Missing or invalid caller credentials receive a fixed `401`; a missing required +server token returns a fixed `503` before the upload body is processed. `GET +/health` remains unauthenticated liveness. `GET /ready` succeeds only when the +authentication configuration and MinerU runtime can accept traffic. + +An isolated local development-only bypass is available only with the explicit +pair below: + +```bash +NEWSDOM_AUTH_MODE=disabled \ +NEWSDOM_RUNTIME_PROFILE=development \ + uv run uvicorn --app-dir src newsdom_api.main:app --reload +``` + +Do not use the development-only bypass as a production rollback. Restore a +working secret or the previous release instead. Each request is written to a request-scoped temporary directory before MinerU runs, and those temporary files are removed after the response completes. diff --git a/docker-compose.yml b/docker-compose.yml index 5f7aba79..19bdffa3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,10 +1,8 @@ # Standalone / sidecar deployment for the NewsDOM PDF-to-DOM service. # -# The default `Dockerfile` ships the API only and does NOT bundle the MinerU -# runtime, so `/parse` returns 503 until a MinerU runtime is available inside -# the image or is pointed to via `NEWSDOM_MINERU_BIN`. The healthcheck below -# targets the unauthenticated `/health` liveness endpoint; note that a green -# `/health` does not by itself guarantee `/parse` readiness (see README). +# This production-oriented example fails before startup when NEWSDOM_API_TOKEN +# is absent. The default API image does not bundle MinerU, so `/ready` remains +# unavailable until a compatible executable is installed or mounted. services: newsdom-api: build: @@ -14,18 +12,19 @@ services: ports: - "8000:8000" environment: - # Optional: point at a bundled/host MinerU executable so /parse can run. + NEWSDOM_AUTH_MODE: required + NEWSDOM_RUNTIME_PROFILE: production + NEWSDOM_API_TOKEN: "${NEWSDOM_API_TOKEN:?set NEWSDOM_API_TOKEN}" + # Point this at a bundled or mounted executable when the default API-only + # image is used. The NVIDIA image already supplies its MinerU runtime. # NEWSDOM_MINERU_BIN: /opt/mineru/bin/mineru - # Optional: when set, /parse requires `Authorization: Bearer `. - # Provide via your orchestrator's secret store rather than inline. - # NEWSDOM_API_TOKEN: ${NEWSDOM_API_TOKEN:-} PYTHONUNBUFFERED: "1" healthcheck: test: - CMD - python3 - -c - - import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health').read() + - import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/ready').read() interval: 30s timeout: 5s start_period: 10s diff --git a/docs/doctoring/fail-closed-parser-authentication.md b/docs/doctoring/fail-closed-parser-authentication.md new file mode 100644 index 00000000..e7df5c28 --- /dev/null +++ b/docs/doctoring/fail-closed-parser-authentication.md @@ -0,0 +1,112 @@ +# Fail-closed parser authentication and readiness + +## Decision + +NewsDOM 0.3.0 changes the parser boundary from **default-open** to +**default-required** authentication. `NEWSDOM_AUTH_MODE` defaults to `required`, +`NEWSDOM_RUNTIME_PROFILE` defaults to `production`, and the application freezes +those values when one FastAPI application instance is created. A missing +`NEWSDOM_API_TOKEN` does not stop process liveness, but it prevents traffic +readiness and blocks `/parse` before multipart upload parsing or MinerU work. + +The only unauthenticated parser bypass is the explicit pair +`NEWSDOM_AUTH_MODE=disabled` and `NEWSDOM_RUNTIME_PROFILE=development`. +Attempting to disable authentication under another profile raises a deterministic +configuration error. The bypass emits one startup warning and never logs a +secret or reversible derivative. + +## Failure domains + +### Caller authentication failure + +A required-mode service with a configured token returns one fixed `401` +response for a missing, malformed, incorrect, non-ASCII, oversized, or duplicated +Authorization header. The response includes `WWW-Authenticate: Bearer`, while +its body contains only `{"detail":"Unauthorized"}`. Raw ASGI header bytes are +compared in constant time against the UTF-8 encoded expected bearer value. + +### Service configuration failure + +A required-mode service without a configured token returns the fixed +`503 Service Unavailable` response from `/parse` before the request body is read. +This is not represented as a caller error because no credential can satisfy an +invalid server configuration. `/ready` reports the same non-sensitive status. + +### Liveness + +`GET /health` proves only that the API process can answer HTTP. It remains +unauthenticated even when authentication or MinerU configuration is unavailable. +Liveness must not cause an orchestrator to restart a healthy process merely +because the service is intentionally refusing traffic. + +### Readiness + +`GET /ready` succeeds only when authentication configuration is valid and the +configured MinerU executable is available. Kubernetes removes a pod from Service +endpoints after readiness failure, so this probe prevents traffic from reaching +a process that cannot safely authorize or parse documents. + +### Development bypass + +The development-only bypass exists for isolated local work where no production +or shared traffic can reach the service. It is never the rollback for a missing +production token. Production rollback restores the previous application release +or secret injection; it does not set `NEWSDOM_AUTH_MODE=disabled`. + +### Standalone and gateway ownership + +The leaf service owns its immutable authentication mode, readiness result, and +constant-time bearer comparison. A naruon or shared API gateway may add stronger +identity, rate limiting, or tenant policy, but the standalone parser still fails +closed if the gateway is bypassed or misconfigured. Gateway health does not +replace the leaf `/ready` contract. + +## Security and operational rationale + +Bearer possession grants access, so RFC 6750 requires protected resources to +support the Authorization header and to return a Bearer challenge when +credentials are absent. Duplicate or malformed credential transport is rejected +rather than normalized. The fixed public responses minimize configuration and +secret disclosure. + +OWASP classifies broken authentication and unrestricted resource consumption as +major API risks. NewsDOM invokes a comparatively expensive document pipeline; +a missing secret therefore creates both an identity failure and an avoidable +compute-cost/denial-of-service boundary. + +Kubernetes distinguishes liveness from readiness: failed readiness removes a pod +from matching Service endpoints while allowing the process to remain alive for +recovery. NewsDOM follows that distinction and does not claim formal compliance +with OWASP, Kubernetes, or OAuth certification programs. + +## Compatibility and release + +The previous default-open behavior is intentionally not preserved. Because the +project remains pre-1.0, the breaking operational default is scheduled for +version **0.3.0** rather than a 1.0 major release. Operators upgrading from 0.2.x +must provision a token, set the explicit production mode/profile, and route +traffic only after `/ready` succeeds. + +## Verification contract + +- required mode with one exact bearer header reaches the parser; +- hostile credential variants return fixed 401 before parser work; +- a missing required token returns fixed 503 before any body read; +- disabled mode is accepted only for development and warns once; +- settings cannot switch after application creation, including under concurrent + requests; +- `/health` and `/ready` remain semantically distinct; +- container and Kubernetes examples bind production mode to a secret reference; +- OpenAPI, logs, errors, and object representations contain no bearer secret; +- production statement and branch coverage remain 100%. + +## References + +Jones, M., & Hardt, D. (2012). *The OAuth 2.0 authorization framework: Bearer +token usage* (RFC 6750). RFC Editor. https://doi.org/10.17487/RFC6750 + +Kubernetes Authors. (2025). *Liveness, readiness, and startup probes*. +Kubernetes Documentation. https://kubernetes.io/docs/concepts/configuration/liveness-readiness-startup-probes/ + +OWASP Foundation. (2023). *OWASP API Security Top 10โ€”2023*. +https://owasp.org/API-Security/editions/2023/en/0x11-t10/ diff --git a/docs/operations/deploy-runbook.md b/docs/operations/deploy-runbook.md index 18075e41..6c481f48 100644 --- a/docs/operations/deploy-runbook.md +++ b/docs/operations/deploy-runbook.md @@ -47,3 +47,33 @@ than a default-image contract. artifacts are missing or invalid. - Reconcile the failure against `README.md`, `CHANGELOG.md`, and the relevant workflow before closing the task. + +## Fail-closed authentication migration for 0.3.0 + +Parser authentication is required by default. The previous default-open behavior +must not be carried into a production or shared deployment. Configure all three +settings before routing traffic: + +```text +NEWSDOM_AUTH_MODE=required +NEWSDOM_RUNTIME_PROFILE=production +NEWSDOM_API_TOKEN= +``` + +Use `GET /health` for process liveness and `GET /ready` for traffic readiness. +A process may be live while `/ready` returns 503 because the required token or +MinerU executable is unavailable. Keep it out of the load-balancer endpoint set +until readiness succeeds. + +The development-only bypass requires both values below and must be confined to +an isolated local workstation: + +```text +NEWSDOM_AUTH_MODE=disabled +NEWSDOM_RUNTIME_PROFILE=development +``` + +This bypass is not a production rollback. To roll back a failed production +upgrade, restore the previous release or repair secret/MinerU injection while +keeping required authentication. Never log the token, its length, or a hash that +could become a reusable credential oracle. diff --git a/docs/operations/kubernetes-deployment.yaml b/docs/operations/kubernetes-deployment.yaml new file mode 100644 index 00000000..1203eb3b --- /dev/null +++ b/docs/operations/kubernetes-deployment.yaml @@ -0,0 +1,65 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: newsdom-api +spec: + replicas: 2 + selector: + matchLabels: + app.kubernetes.io/name: newsdom-api + template: + metadata: + labels: + app.kubernetes.io/name: newsdom-api + spec: + containers: + - name: newsdom-api + image: ghcr.io/contextualwisdomlab/newsdom-api:0.3.0 + ports: + - name: http + containerPort: 8000 + env: + - name: NEWSDOM_AUTH_MODE + value: required + - name: NEWSDOM_RUNTIME_PROFILE + value: production + - name: NEWSDOM_API_TOKEN + valueFrom: + secretKeyRef: + name: newsdom-api-runtime + key: api_token + - name: NEWSDOM_MINERU_BIN + value: /opt/mineru/bin/mineru + livenessProbe: + httpGet: + path: /health + port: http + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /ready + port: http + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: "2" + memory: 4Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: newsdom-api +spec: + selector: + app.kubernetes.io/name: newsdom-api + ports: + - name: http + port: 8000 + targetPort: http diff --git a/src/newsdom_api/config.py b/src/newsdom_api/config.py index 7daf6dc8..fc073033 100644 --- a/src/newsdom_api/config.py +++ b/src/newsdom_api/config.py @@ -1,31 +1,134 @@ -"""Bootstrap configuration for the standalone NewsDOM sidecar service. - -This module reads the leaf service's *own* runtime settings from the process -environment. It is deliberately minimal: the environment is used only as a -bootstrap transport for the sidecar's own configuration (mirroring how -``NEWSDOM_MINERU_BIN`` is resolved in :mod:`newsdom_api.mineru_runner`). No -secret is hardcoded, and values are evaluated on each call so deployments can -rotate them without a process restart. -""" +"""Immutable runtime configuration for the standalone NewsDOM sidecar.""" from __future__ import annotations import os +from dataclasses import dataclass, field +from enum import Enum +from typing import Mapping -# Environment variable holding the optional shared bearer secret for ``/parse``. API_TOKEN_ENV_VAR = "NEWSDOM_API_TOKEN" +AUTH_MODE_ENV_VAR = "NEWSDOM_AUTH_MODE" +RUNTIME_PROFILE_ENV_VAR = "NEWSDOM_RUNTIME_PROFILE" +MAX_BEARER_HEADER_BYTES = 4096 + + +class RuntimeConfigurationError(ValueError): + """Raised when runtime settings violate a fail-closed service boundary.""" + + +class AuthenticationMode(str, Enum): + """Supported parser authentication modes.""" + + REQUIRED = "required" + DISABLED = "disabled" + + +class RuntimeProfile(str, Enum): + """Runtime profiles that determine whether development bypasses are valid.""" + + PRODUCTION = "production" + DEVELOPMENT = "development" + + +@dataclass(frozen=True, slots=True) +class RuntimeSettings: + """Validated settings frozen for the lifetime of one FastAPI application.""" + + authentication_mode: AuthenticationMode = AuthenticationMode.REQUIRED + runtime_profile: RuntimeProfile = RuntimeProfile.PRODUCTION + api_token: str | None = field(default=None, repr=False) + def __post_init__(self) -> None: + """Normalize secrets once and reject unsafe direct construction.""" -def get_api_token() -> str | None: - """Return the configured ``/parse`` bearer token, or ``None`` when unset. + if ( + self.authentication_mode is AuthenticationMode.DISABLED + and self.runtime_profile is not RuntimeProfile.DEVELOPMENT + ): + raise RuntimeConfigurationError( + "Authentication can be disabled only in the development runtime profile" + ) - When the variable is absent or blank the service stays open (development - default). Any surrounding whitespace is stripped so an accidentally quoted - or newline-terminated secret does not silently disable auth comparison. - """ + if self.api_token is None: + return - raw = os.environ.get(API_TOKEN_ENV_VAR) + normalized_token = self.api_token.strip() + if not normalized_token: + raise RuntimeConfigurationError( + "The configured parser authentication token must not be blank" + ) + bearer_value = f"Bearer {normalized_token}".encode("utf-8") + if len(bearer_value) > MAX_BEARER_HEADER_BYTES: + raise RuntimeConfigurationError( + "The configured parser authentication token is too long" + ) + object.__setattr__(self, "api_token", normalized_token) + + @property + def authentication_ready(self) -> bool: + """Return whether the authentication configuration can serve traffic safely.""" + + return ( + self.authentication_mode is AuthenticationMode.DISABLED + or self.api_token is not None + ) + + +def _parse_enum_value( + source: Mapping[str, str], + variable: str, + enum_type: type[Enum], + default: str, +) -> str: + """Parse one normalized enum setting or raise a non-sensitive error.""" + + raw = source.get(variable, default).strip().lower() + try: + enum_type(raw) + except ValueError as exc: + raise RuntimeConfigurationError( + f"Invalid value for {variable}; use one of the documented modes" + ) from exc + return raw + + +def get_api_token(source: Mapping[str, str] | None = None) -> str | None: + """Return a normalized bootstrap token without logging or exposing it.""" + + values = os.environ if source is None else source + raw = values.get(API_TOKEN_ENV_VAR) if raw is None: return None token = raw.strip() return token or None + + +def load_runtime_settings( + source: Mapping[str, str] | None = None, +) -> RuntimeSettings: + """Load and validate a single immutable application configuration snapshot.""" + + values = os.environ if source is None else source + authentication_mode = AuthenticationMode( + _parse_enum_value( + values, + AUTH_MODE_ENV_VAR, + AuthenticationMode, + AuthenticationMode.REQUIRED.value, + ) + ) + runtime_profile = RuntimeProfile( + _parse_enum_value( + values, + RUNTIME_PROFILE_ENV_VAR, + RuntimeProfile, + RuntimeProfile.PRODUCTION.value, + ) + ) + + return RuntimeSettings( + authentication_mode=authentication_mode, + runtime_profile=runtime_profile, + api_token=get_api_token(values), + ) diff --git a/src/newsdom_api/main.py b/src/newsdom_api/main.py index ca5b7599..6723284a 100644 --- a/src/newsdom_api/main.py +++ b/src/newsdom_api/main.py @@ -14,69 +14,51 @@ FastAPI, File, Form, - Header, HTTPException, Request, Response, UploadFile, ) from fastapi.responses import JSONResponse +from fastapi.security import HTTPBearer from pypdf import PdfReader from pypdf.errors import PdfReadError -from .config import get_api_token +from .config import ( + AuthenticationMode, + MAX_BEARER_HEADER_BYTES, + RuntimeSettings, + load_runtime_settings, +) from .errors import MineruIncompleteOutputError, MineruRuntimeUnavailableError from .mineru_runner import ( DEFAULT_LANGUAGE, DEFAULT_MODE, + mineru_runtime_available, normalize_language, normalize_mode, ) -from .schemas import HealthResponse, ParseResponse +from .schemas import HealthResponse, ParseResponse, ReadinessResponse from .service import parse_pdf MAX_PARSE_UPLOAD_BYTES = 20 * 1024 * 1024 +MAX_AUTHORIZATION_HEADER_BYTES = MAX_BEARER_HEADER_BYTES UNSUPPORTED_MEDIA_DETAIL = "Unsupported Media Type" PAYLOAD_TOO_LARGE_DETAIL = "Payload Too Large" INVALID_PARSE_PARAMS_DETAIL = "Invalid parse parameters" UNAUTHORIZED_DETAIL = "Unauthorized" +SERVICE_UNAVAILABLE_DETAIL = "Service Unavailable" LOGGER = logging.getLogger("newsdom_api") +BEARER_SCHEME = HTTPBearer(auto_error=False, scheme_name="BearerAuth") tags_metadata = [ - { - "name": "Parser", - "description": "Core PDF parsing endpoints.", - }, + {"name": "Parser", "description": "Core PDF parsing endpoints."}, { "name": "System", "description": "Health and deployment diagnostic endpoints.", }, ] -app = FastAPI( - title="NewsDOM API", - description=( - "Language-agnostic PDF-to-DOM parser API. Converts a PDF into a " - "canonical JSON document tree (pages, sections, headings, body blocks, " - "images, and bounding boxes) using MinerU." - ), - version="0.2.0", - contact={ - "name": "Seongho Bae", - "url": "https://github.com/Seongho-Bae/newsdom-api", - }, - license_info={ - "name": "MIT License", - "identifier": "MIT", - }, - openapi_tags=tags_metadata, - swagger_ui_parameters={ - "displayRequestDuration": True, - "syntaxHighlight.theme": "monokai", - "tryItOutEnabled": True, - }, -) - def _apply_security_headers(response: Response, request: Request) -> Response: """Inject standard security headers into an API response.""" @@ -97,14 +79,75 @@ def _apply_security_headers(response: Response, request: Request) -> Response: return response -@app.middleware("http") -async def add_security_headers(request: Request, call_next: Callable) -> Response: - """Inject standard security headers into all API responses.""" +def _runtime_settings(request: Request) -> RuntimeSettings: + """Return the immutable settings bound to the current application instance.""" + + settings = request.app.state.runtime_settings + if not isinstance(settings, RuntimeSettings): + raise RuntimeError("Application runtime settings are unavailable") + return settings + + +def _authorization_values(request: Request) -> list[bytes]: + """Return every raw Authorization header without lossy string coercion.""" + + return [ + value + for name, value in request.scope.get("headers", []) + if name.lower() == b"authorization" + ] + + +def _unauthorized_response() -> JSONResponse: + """Return one fixed caller-authentication failure response.""" + + return JSONResponse( + status_code=401, + content={"detail": UNAUTHORIZED_DETAIL}, + headers={"WWW-Authenticate": "Bearer"}, + ) + + +def _parse_access_failure(request: Request) -> JSONResponse | None: + """Validate `/parse` authorization before multipart upload parsing begins.""" + + settings = _runtime_settings(request) + if settings.authentication_mode is AuthenticationMode.DISABLED: + return None + token = settings.api_token + if token is None: + return JSONResponse( + status_code=503, + content={"detail": SERVICE_UNAVAILABLE_DETAIL}, + ) + + authorization_values = _authorization_values(request) + if len(authorization_values) != 1: + return _unauthorized_response() + provided = authorization_values[0] + if len(provided) > MAX_BEARER_HEADER_BYTES: + return _unauthorized_response() + + expected = f"Bearer {token}".encode("utf-8") + if not hmac.compare_digest(provided, expected): + return _unauthorized_response() + return None + + +async def security_boundary_middleware( + request: Request, + call_next: Callable, +) -> Response: + """Enforce parser authorization before reading the request body and add headers.""" + + if request.method == "POST" and request.scope.get("path") == "/parse": + failure = _parse_access_failure(request) + if failure is not None: + return _apply_security_headers(failure, request) response = await call_next(request) return _apply_security_headers(response, request) -@app.exception_handler(Exception) async def global_exception_handler(request: Request, exc: Exception) -> Response: """Return sanitized 500 responses with the standard security headers.""" @@ -116,53 +159,29 @@ async def global_exception_handler(request: Request, exc: Exception) -> Response return _apply_security_headers(response, request) -def require_authorization( - authorization: Annotated[str | None, Header()] = None, -) -> None: - """Enforce optional bearer authentication on protected endpoints. +def health() -> HealthResponse: + """Return a minimal liveness response independent from service readiness.""" - When the sidecar has a shared secret configured (via - :func:`newsdom_api.config.get_api_token`), callers must present a matching - ``Authorization: Bearer `` header; otherwise a ``401`` is raised. If - no secret is configured the service stays open, which keeps the standalone - development experience friction-free. The comparison is constant-time. - """ + return HealthResponse() - token = get_api_token() - if token is None: - return - expected = f"Bearer {token}" - provided = authorization or "" - if not hmac.compare_digest(provided.encode("utf-8"), expected.encode("utf-8")): - raise HTTPException( - status_code=401, - detail=UNAUTHORIZED_DETAIL, - headers={"WWW-Authenticate": "Bearer"}, - ) +def ready(request: Request) -> ReadinessResponse: + """Return readiness only when authentication and MinerU runtime are available.""" -@app.get( - "/health", - response_model=HealthResponse, - summary="Health Check", - description="Returns a minimal liveness response for deployment health checks.", - tags=["System"], -) -def health() -> HealthResponse: - """Return a minimal liveness response for health checks.""" - - return HealthResponse() + settings = _runtime_settings(request) + runtime_probe = request.app.state.runtime_readiness_probe + if not settings.authentication_ready or not runtime_probe(): + raise HTTPException(status_code=503, detail=SERVICE_UNAVAILABLE_DETAIL) + return ReadinessResponse() def _validate_pdf_structure(file_path: Path) -> None: """Reject payloads that are not structurally parseable PDFs.""" - with file_path.open("rb") as f: - magic = f.read(5) + + with file_path.open("rb") as file_handle: + magic = file_handle.read(5) if magic != b"%PDF-": - raise HTTPException( - status_code=415, - detail=UNSUPPORTED_MEDIA_DETAIL, - ) + raise HTTPException(status_code=415, detail=UNSUPPORTED_MEDIA_DETAIL) try: reader = PdfReader(file_path, strict=True) if len(reader.pages) < 1: @@ -174,28 +193,6 @@ def _validate_pdf_structure(file_path: Path) -> None: ) from None -@app.post( - "/parse", - response_model=ParseResponse, - summary="Parse PDF into a DOM tree", - description=( - "Converts a PDF into a canonical JSON DOM document using MinerU. The " - "optional `language` and `mode` form fields select the MinerU language " - "family (default `ch`) and parsing mode (`auto`/`ocr`/`txt`, " - "default `auto`). When a shared secret is configured, an " - "`Authorization: Bearer ` header is required." - ), - dependencies=[Depends(require_authorization)], - responses={ - 401: {"description": "Unauthorized"}, - 413: {"description": "Payload Too Large"}, - 415: {"description": "Unsupported Media Type"}, - 422: {"description": "Invalid parse parameters"}, - 502: {"description": "Bad Gateway"}, - 503: {"description": "Service Unavailable"}, - }, - tags=["Parser"], -) async def parse( file: Annotated[UploadFile, File(..., description="The PDF file to parse.")], language: Annotated[ @@ -217,7 +214,7 @@ async def parse( ), ] = DEFAULT_MODE, ) -> ParseResponse: - """Parse an uploaded PDF into the canonical DOM response model.""" + """Parse an authorized uploaded PDF into the canonical DOM response model.""" media_type = (file.content_type or "").split(";", 1)[0].strip().lower() if media_type != "application/pdf": @@ -228,7 +225,8 @@ async def parse( resolved_mode = normalize_mode(mode) except ValueError: raise HTTPException( - status_code=422, detail=INVALID_PARSE_PARAMS_DETAIL + status_code=422, + detail=INVALID_PARSE_PARAMS_DETAIL, ) from None file_size = getattr(file, "size", None) @@ -239,27 +237,26 @@ async def parse( try: header = await file.read(5) if header != b"%PDF-": - raise HTTPException( - status_code=415, - detail=UNSUPPORTED_MEDIA_DETAIL, - ) + raise HTTPException(status_code=415, detail=UNSUPPORTED_MEDIA_DETAIL) - with tempfile.NamedTemporaryFile(delete=False) as tmp: - tmp_path = Path(tmp.name) + with tempfile.NamedTemporaryFile(delete=False) as temporary_file: + tmp_path = Path(temporary_file.name) LOGGER.debug("Created temporary upload file %s", tmp_path) - tmp.write(header) + temporary_file.write(header) bytes_read = len(header) while chunk := await file.read(8192): bytes_read += len(chunk) if bytes_read > MAX_PARSE_UPLOAD_BYTES: LOGGER.warning( - "Rejecting upload over limit: %s bytes read", bytes_read + "Rejecting upload over limit: %s bytes read", + bytes_read, ) raise HTTPException( - status_code=413, detail=PAYLOAD_TOO_LARGE_DETAIL + status_code=413, + detail=PAYLOAD_TOO_LARGE_DETAIL, ) - tmp.write(chunk) + temporary_file.write(chunk) LOGGER.debug("Wrote %s upload bytes to %s", bytes_read, tmp_path) _validate_pdf_structure(tmp_path) @@ -270,9 +267,10 @@ async def parse( language=resolved_language, mode=resolved_mode, ) - except MineruRuntimeUnavailableError: - raise HTTPException(status_code=503, detail="Service Unavailable") from None + raise HTTPException( + status_code=503, detail=SERVICE_UNAVAILABLE_DETAIL + ) from None except MineruIncompleteOutputError: raise HTTPException(status_code=502, detail="Bad Gateway") from None finally: @@ -281,3 +279,93 @@ async def parse( tmp_path.unlink(missing_ok=True) except OSError: LOGGER.exception("Failed to remove temporary upload file %s", tmp_path) + + +def create_app( + settings: RuntimeSettings | None = None, + *, + runtime_readiness_probe: Callable[[], bool] | None = None, +) -> FastAPI: + """Create a NewsDOM application with immutable settings and injected readiness.""" + + application_settings = settings or load_runtime_settings() + if application_settings.authentication_mode is AuthenticationMode.DISABLED: + LOGGER.warning( + "Parser authentication is disabled for the explicit development profile" + ) + elif not application_settings.authentication_ready: + LOGGER.error("Parser authentication configuration is unavailable") + + application = FastAPI( + title="NewsDOM API", + description=( + "Language-agnostic PDF-to-DOM parser API. Converts a PDF into a " + "canonical JSON document tree using MinerU. Authentication is " + "required by default; an explicit development-only bypass is available." + ), + version="0.2.0", + contact={ + "name": "Seongho Bae", + "url": "https://github.com/ContextualWisdomLab/newsdom-api", + }, + license_info={"name": "MIT License", "identifier": "MIT"}, + openapi_tags=tags_metadata, + swagger_ui_parameters={ + "displayRequestDuration": True, + "syntaxHighlight.theme": "monokai", + "tryItOutEnabled": True, + }, + ) + application.state.runtime_settings = application_settings + application.state.runtime_readiness_probe = ( + runtime_readiness_probe or mineru_runtime_available + ) + application.middleware("http")(security_boundary_middleware) + application.add_exception_handler(Exception, global_exception_handler) + application.add_api_route( + "/health", + health, + methods=["GET"], + response_model=HealthResponse, + summary="Health Check", + description="Returns liveness without evaluating traffic readiness.", + tags=["System"], + ) + application.add_api_route( + "/ready", + ready, + methods=["GET"], + response_model=ReadinessResponse, + summary="Readiness Check", + description=( + "Returns ready only when parser authentication configuration and " + "the MinerU runtime can safely accept traffic." + ), + responses={503: {"description": SERVICE_UNAVAILABLE_DETAIL}}, + tags=["System"], + ) + application.add_api_route( + "/parse", + parse, + methods=["POST"], + response_model=ParseResponse, + summary="Parse PDF into a DOM tree", + description=( + "Converts an authenticated PDF upload into canonical NewsDOM JSON. " + "The optional language and mode form fields configure MinerU." + ), + dependencies=[Depends(BEARER_SCHEME)], + responses={ + 401: {"description": UNAUTHORIZED_DETAIL}, + 413: {"description": PAYLOAD_TOO_LARGE_DETAIL}, + 415: {"description": UNSUPPORTED_MEDIA_DETAIL}, + 422: {"description": INVALID_PARSE_PARAMS_DETAIL}, + 502: {"description": "Bad Gateway"}, + 503: {"description": SERVICE_UNAVAILABLE_DETAIL}, + }, + tags=["Parser"], + ) + return application + + +app = create_app() diff --git a/src/newsdom_api/mineru_runner.py b/src/newsdom_api/mineru_runner.py index 68bba5bd..45e74fe3 100644 --- a/src/newsdom_api/mineru_runner.py +++ b/src/newsdom_api/mineru_runner.py @@ -214,6 +214,19 @@ def _resolve_mineru_bin() -> str: return found +def mineru_runtime_available() -> bool: + """Return whether the configured MinerU executable resolves for this process.""" + + try: + executable = _resolve_mineru_bin() + except MineruRuntimeUnavailableError: + return False + if os.path.sep in executable or (os.path.altsep and os.path.altsep in executable): + path = Path(executable) + return path.is_file() and os.access(path, os.X_OK) + return _cached_which(executable) is not None + + def _find_output_dir(base_output_dir: Path, method: str = DEFAULT_MODE) -> Path: """Locate the parse-method output directory created by MinerU. diff --git a/src/newsdom_api/schemas.py b/src/newsdom_api/schemas.py index e0d0ddf8..51703810 100644 --- a/src/newsdom_api/schemas.py +++ b/src/newsdom_api/schemas.py @@ -37,7 +37,9 @@ class CaptionNode(BaseModel): ) bbox: Optional[BoundingBox] = Field( default=None, - description="Bounding box of the caption when parser coordinates are available.", + description=( + "Bounding box of the caption when parser coordinates are available." + ), ) @@ -88,7 +90,9 @@ class ArticleNode(BaseModel): ) bbox: Optional[BoundingBox] = Field( default=None, - description="Bounding box enclosing the article when parser coordinates are available.", + description=( + "Bounding box enclosing the article when parser coordinates are available." + ), ) body_blocks: List[str] = Field( default_factory=list, @@ -190,3 +194,13 @@ class HealthResponse(BaseModel): description="Current operational status of the service.", json_schema_extra={"example": "ok"}, ) + + +class ReadinessResponse(BaseModel): + """Traffic-readiness response emitted only when all required dependencies work.""" + + status: str = Field( + default="ready", + description="Stable traffic-readiness status.", + json_schema_extra={"example": "ready"}, + ) diff --git a/tests/conftest.py b/tests/conftest.py index 8a23fa03..18cfdca1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,8 +1,15 @@ from __future__ import annotations +import os import sys from pathlib import Path +# Existing endpoint tests intentionally run under the explicit development-only +# bypass. Production defaults remain fail-closed and are covered through the +# application factory with isolated RuntimeSettings instances. +os.environ.setdefault("NEWSDOM_AUTH_MODE", "disabled") +os.environ.setdefault("NEWSDOM_RUNTIME_PROFILE", "development") + ROOT = Path(__file__).resolve().parents[1] SRC = ROOT / "src" diff --git a/tests/test_auth.py b/tests/test_auth.py index 88cf6412..055d23e8 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -1,105 +1,413 @@ -"""Tests for the optional bearer-auth gate on ``/parse`` and the leaf config.""" +"""Tests for fail-closed parser authentication and immutable runtime settings.""" from __future__ import annotations +import asyncio +import logging +import os +from concurrent.futures import ThreadPoolExecutor + import pytest +from fastapi import Request from fastapi.testclient import TestClient from newsdom_api import config -from newsdom_api.config import API_TOKEN_ENV_VAR, get_api_token -from newsdom_api.main import app +from newsdom_api.config import ( + API_TOKEN_ENV_VAR, + AUTH_MODE_ENV_VAR, + RUNTIME_PROFILE_ENV_VAR, + AuthenticationMode, + RuntimeConfigurationError, + RuntimeProfile, + RuntimeSettings, + get_api_token, + load_runtime_settings, +) +from newsdom_api.main import ( + MAX_AUTHORIZATION_HEADER_BYTES, + create_app, + security_boundary_middleware, +) + +_PDF_FILES = { + "file": ("fixture.pdf", b"%PDF-1.4\n%synthetic\n", "application/pdf") +} + -_PDF_FILES = {"file": ("fixture.pdf", b"%PDF-1.4\n%synthetic\n", "application/pdf")} +def _settings( + *, + token: str | None = "s3cret-token", + mode: AuthenticationMode = AuthenticationMode.REQUIRED, + profile: RuntimeProfile = RuntimeProfile.PRODUCTION, +) -> RuntimeSettings: + """Build one explicit immutable settings object for an application test.""" + + return RuntimeSettings( + authentication_mode=mode, + runtime_profile=profile, + api_token=token, + ) @pytest.fixture -def stub_parser(monkeypatch): - """Bypass PDF structure validation and MinerU so auth is what is exercised.""" +def parser_spy(monkeypatch: pytest.MonkeyPatch) -> dict[str, int]: + """Replace PDF validation and parsing while recording real parser entry.""" + + calls = {"count": 0} - def fake_parse_pdf(file_path, filename, **kwargs): + def fake_parse_pdf(*_args, **_kwargs): + calls["count"] += 1 return {"document_id": "fixture", "pages": []} monkeypatch.setattr("newsdom_api.main._validate_pdf_structure", lambda _: None) monkeypatch.setattr("newsdom_api.main.parse_pdf", fake_parse_pdf) + return calls -def test_parse_is_open_when_no_secret_configured(monkeypatch, stub_parser): - monkeypatch.delenv(API_TOKEN_ENV_VAR, raising=False) - client = TestClient(app) - response = client.post("/parse", files=_PDF_FILES) +def test_required_mode_accepts_exact_bearer(parser_spy: dict[str, int]) -> None: + """A configured required-mode service should accept one exact bearer header.""" + + application = create_app(_settings(), runtime_readiness_probe=lambda: True) + response = TestClient(application).post( + "/parse", + files=_PDF_FILES, + headers={"Authorization": "Bearer s3cret-token"}, + ) + assert response.status_code == 200 + assert parser_spy["count"] == 1 -def test_parse_requires_bearer_when_secret_set_and_header_missing( - monkeypatch, stub_parser -): - monkeypatch.setenv(API_TOKEN_ENV_VAR, "s3cret-token") - client = TestClient(app) - response = client.post("/parse", files=_PDF_FILES) +@pytest.mark.parametrize( + "headers", + [ + None, + {"Authorization": "Bearer wrong-token"}, + [(b"Authorization", b"Bearer \xe2\x98\x83")], + {"Authorization": "Basic dXNlcjpwYXNz"}, + {"Authorization": "Bearer " + "x" * (MAX_AUTHORIZATION_HEADER_BYTES + 1)}, + [ + (b"Authorization", b"Bearer s3cret-token"), + (b"Authorization", b"Bearer s3cret-token"), + ], + ], +) +def test_required_mode_rejects_hostile_credentials_before_parser( + headers, + parser_spy: dict[str, int], +) -> None: + """Missing, malformed, Unicode, oversized, or duplicated credentials stay 401.""" + + application = create_app(_settings(), runtime_readiness_probe=lambda: True) + request_kwargs = {"files": _PDF_FILES} + if headers is not None: + request_kwargs["headers"] = headers + response = TestClient(application, raise_server_exceptions=False).post( + "/parse", **request_kwargs + ) + assert response.status_code == 401 - assert response.json()["detail"] == "Unauthorized" - assert response.headers.get("WWW-Authenticate") == "Bearer" + assert response.json() == {"detail": "Unauthorized"} + assert response.headers["WWW-Authenticate"] == "Bearer" + assert parser_spy["count"] == 0 -def test_parse_rejects_invalid_bearer_when_secret_set(monkeypatch, stub_parser): - monkeypatch.setenv(API_TOKEN_ENV_VAR, "s3cret-token") - client = TestClient(app) - response = client.post( - "/parse", - files=_PDF_FILES, - headers={"Authorization": "Bearer wrong-token"}, +def test_missing_required_token_is_service_configuration_failure( + parser_spy: dict[str, int], +) -> None: + """A missing server token should return fixed 503 rather than caller 401.""" + + application = create_app( + _settings(token=None), runtime_readiness_probe=lambda: True ) - assert response.status_code == 401 - assert response.json()["detail"] == "Unauthorized" + response = TestClient(application).post("/parse", files=_PDF_FILES) + assert response.status_code == 503 + assert response.json() == {"detail": "Service Unavailable"} + assert "WWW-Authenticate" not in response.headers + assert parser_spy["count"] == 0 -def test_parse_rejects_non_ascii_bearer_when_secret_set(monkeypatch, stub_parser): - monkeypatch.setenv(API_TOKEN_ENV_VAR, "s3cret-token") - client = TestClient(app, raise_server_exceptions=False) - # The ASGI spec and TestClient headers require raw byte tuples to inject non-ASCII - # natively without framework-level pre-validation failures. - response = client.post( - "/parse", - files=_PDF_FILES, - headers=[(b"Authorization", b"Bearer \xe2\x98\x83")], + +def test_development_disabled_mode_allows_parse_and_warns_once( + parser_spy: dict[str, int], caplog: pytest.LogCaptureFixture +) -> None: + """The explicit development bypass should be usable and logged once at startup.""" + + caplog.set_level(logging.WARNING, logger="newsdom_api") + application = create_app( + _settings( + token=None, + mode=AuthenticationMode.DISABLED, + profile=RuntimeProfile.DEVELOPMENT, + ), + runtime_readiness_probe=lambda: True, ) - assert response.status_code == 401 - assert response.json()["detail"] == "Unauthorized" + client = TestClient(application) + assert client.post("/parse", files=_PDF_FILES).status_code == 200 + assert client.post("/parse", files=_PDF_FILES).status_code == 200 + assert parser_spy["count"] == 2 + messages = [record.getMessage() for record in caplog.records] + assert messages.count( + "Parser authentication is disabled for the explicit development profile" + ) == 1 -def test_parse_accepts_valid_bearer_when_secret_set(monkeypatch, stub_parser): - monkeypatch.setenv(API_TOKEN_ENV_VAR, "s3cret-token") - client = TestClient(app) - response = client.post( - "/parse", - files=_PDF_FILES, - headers={"Authorization": "Bearer s3cret-token"}, + +def test_direct_runtime_settings_reject_invalid_security_invariants() -> None: + """Direct application settings must enforce the same fail-closed boundary.""" + + with pytest.raises(RuntimeConfigurationError, match="development"): + RuntimeSettings( + authentication_mode=AuthenticationMode.DISABLED, + runtime_profile=RuntimeProfile.PRODUCTION, + api_token=None, + ) + + with pytest.raises(RuntimeConfigurationError, match="blank"): + RuntimeSettings(api_token=" ") + + with pytest.raises(RuntimeConfigurationError, match="too long"): + RuntimeSettings(api_token="x" * 4090) + +def test_direct_runtime_settings_normalize_token_once() -> None: + """Direct construction should freeze the normalized bearer secret.""" + + settings = RuntimeSettings(api_token=" stable-token\n") + + assert settings.api_token == "stable-token" + assert settings.authentication_ready is True + + +def test_disabled_mode_is_rejected_outside_development() -> None: + """Production cannot opt into the unauthenticated development bypass.""" + + with pytest.raises(RuntimeConfigurationError, match="development"): + load_runtime_settings( + { + AUTH_MODE_ENV_VAR: "disabled", + RUNTIME_PROFILE_ENV_VAR: "production", + } + ) + + +@pytest.mark.parametrize( + ("variable", "value"), + [ + (AUTH_MODE_ENV_VAR, "sometimes"), + (RUNTIME_PROFILE_ENV_VAR, "staging"), + ], +) +def test_unknown_runtime_modes_fail_deterministically( + variable: str, value: str +) -> None: + """Unknown mode strings must not silently select a permissive fallback.""" + + with pytest.raises(RuntimeConfigurationError, match=variable): + load_runtime_settings({variable: value}) + + +def test_runtime_settings_are_immutable_after_application_creation( + monkeypatch: pytest.MonkeyPatch, parser_spy: dict[str, int] +) -> None: + """Environment changes after app creation must not rotate its security state.""" + + application = create_app( + _settings(token="token-a"), runtime_readiness_probe=lambda: True ) - assert response.status_code == 200 + monkeypatch.setenv(API_TOKEN_ENV_VAR, "token-b") + client = TestClient(application) + rejected = client.post( + "/parse", files=_PDF_FILES, headers={"Authorization": "Bearer token-b"} + ) + accepted = client.post( + "/parse", files=_PDF_FILES, headers={"Authorization": "Bearer token-a"} + ) + + assert rejected.status_code == 401 + assert accepted.status_code == 200 + assert parser_spy["count"] == 1 + + + +def test_corrupted_application_settings_fail_closed_with_sanitized_response() -> None: + """Missing runtime state must not expose internals or accept traffic.""" + + application = create_app(_settings(), runtime_readiness_probe=lambda: True) + application.state.runtime_settings = object() + + response = TestClient(application, raise_server_exceptions=False).get( + "/ready" + ) + + assert response.status_code == 500 + assert response.json() == {"detail": "Internal Server Error"} + assert response.headers["X-Content-Type-Options"] == "nosniff" + +def test_health_is_always_liveness_only() -> None: + """Health should remain public even when authentication readiness is invalid.""" + + application = create_app( + _settings(token=None), runtime_readiness_probe=lambda: False + ) + response = TestClient(application).get("/health") -def test_health_is_unauthenticated_even_when_secret_set(monkeypatch): - monkeypatch.setenv(API_TOKEN_ENV_VAR, "s3cret-token") - client = TestClient(app) - response = client.get("/health") assert response.status_code == 200 assert response.json() == {"status": "ok"} -def test_get_api_token_returns_none_when_unset(monkeypatch): - monkeypatch.delenv(API_TOKEN_ENV_VAR, raising=False) - assert get_api_token() is None +@pytest.mark.parametrize( + ("settings", "runtime_ready", "expected_status"), + [ + (_settings(), True, 200), + (_settings(token=None), True, 503), + (_settings(), False, 503), + ( + _settings( + token=None, + mode=AuthenticationMode.DISABLED, + profile=RuntimeProfile.DEVELOPMENT, + ), + True, + 200, + ), + ], +) +def test_ready_combines_authentication_and_parser_runtime( + settings: RuntimeSettings, + runtime_ready: bool, + expected_status: int, +) -> None: + """Readiness must represent both security configuration and parser runtime.""" + + application = create_app( + settings, runtime_readiness_probe=lambda: runtime_ready + ) + response = TestClient(application).get("/ready") + + assert response.status_code == expected_status + if expected_status == 200: + assert response.json() == {"status": "ready"} + else: + assert response.json() == {"detail": "Service Unavailable"} + assert settings.api_token not in response.text if settings.api_token else True + + +def test_default_runtime_probe_is_not_an_unconditional_success( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The default app factory must consult MinerU availability rather than a stub.""" + + monkeypatch.setattr( + "newsdom_api.main.mineru_runtime_available", lambda: False, raising=False + ) + response = TestClient(create_app(_settings())).get("/ready") + + assert response.status_code == 503 + + +def test_authentication_middleware_rejects_before_reading_request_body() -> None: + """Reject parser access before ASGI body receive or downstream parsing.""" + + application = create_app( + _settings(token=None), runtime_readiness_probe=lambda: True + ) + body_read = False + + async def receive(): + nonlocal body_read + body_read = True + raise AssertionError("request body should not be read") + + async def call_next(_request): + raise AssertionError("downstream application should not run") + + scope = { + "type": "http", + "asgi": {"version": "3.0"}, + "http_version": "1.1", + "method": "POST", + "scheme": "http", + "path": "/parse", + "raw_path": b"/parse", + "query_string": b"", + "headers": [], + "client": ("testclient", 50000), + "server": ("testserver", 80), + "app": application, + } + request = Request(scope, receive) + + response = asyncio.run(security_boundary_middleware(request, call_next)) + + assert response.status_code == 503 + assert body_read is False + + +def test_concurrent_requests_cannot_switch_authentication_state( + parser_spy: dict[str, int], +) -> None: + """Concurrent callers should observe one frozen token and mode.""" + + application = create_app( + _settings(token="fixed"), runtime_readiness_probe=lambda: True + ) + + def request(token: str) -> int: + return TestClient(application).post( + "/parse", + files=_PDF_FILES, + headers={"Authorization": f"Bearer {token}"}, + ).status_code + + tokens = ["fixed" if index % 2 == 0 else "changed" for index in range(20)] + with ThreadPoolExecutor(max_workers=8) as executor: + statuses = list(executor.map(request, tokens)) + + assert statuses.count(200) == 10 + assert statuses.count(401) == 10 + assert parser_spy["count"] == 10 + + +def test_openapi_and_settings_repr_do_not_expose_bearer_secret() -> None: + """Schema output and diagnostic repr must never contain the configured token.""" + + secret = "do-not-disclose" + settings = _settings(token=secret) + application = create_app(settings, runtime_readiness_probe=lambda: True) + openapi_response = TestClient(application).get("/openapi.json") + openapi_text = openapi_response.text + openapi_document = openapi_response.json() + + assert secret not in openapi_text + assert secret not in repr(settings) + assert openapi_document["components"]["securitySchemes"]["BearerAuth"] == { + "type": "http", + "scheme": "bearer", + } + + +def test_get_api_token_normalizes_bootstrap_transport() -> None: + """Whitespace-only secrets are absent and mounted newline suffixes are stripped.""" + + assert get_api_token({}) is None + assert get_api_token({API_TOKEN_ENV_VAR: " "}) is None + assert get_api_token({API_TOKEN_ENV_VAR: " padded-token\n"}) == "padded-token" + +def test_default_runtime_settings_are_required_production() -> None: + """Missing mode variables must select the fail-closed production contract.""" -def test_get_api_token_treats_blank_as_disabled(monkeypatch): - monkeypatch.setenv(API_TOKEN_ENV_VAR, " ") - assert get_api_token() is None + settings = load_runtime_settings({API_TOKEN_ENV_VAR: "token"}) + assert settings.authentication_mode is AuthenticationMode.REQUIRED + assert settings.runtime_profile is RuntimeProfile.PRODUCTION + assert settings.authentication_ready is True -def test_get_api_token_strips_surrounding_whitespace(monkeypatch): - monkeypatch.setenv(API_TOKEN_ENV_VAR, " padded-token\n") - assert get_api_token() == "padded-token" +def test_config_module_exposes_versioned_environment_contract() -> None: + """Deployment tooling should use the exact documented environment names.""" -def test_config_module_exposes_env_var_name(): assert config.API_TOKEN_ENV_VAR == "NEWSDOM_API_TOKEN" + assert config.AUTH_MODE_ENV_VAR == "NEWSDOM_AUTH_MODE" + assert config.RUNTIME_PROFILE_ENV_VAR == "NEWSDOM_RUNTIME_PROFILE" diff --git a/tests/test_auth_deployment_contract.py b/tests/test_auth_deployment_contract.py new file mode 100644 index 00000000..eb0fe7aa --- /dev/null +++ b/tests/test_auth_deployment_contract.py @@ -0,0 +1,111 @@ +"""Deployment and documentation contracts for fail-closed authentication.""" + +from __future__ import annotations + +from pathlib import Path + +import yaml + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] + + +def _text(path: str) -> str: + """Read one repository-relative UTF-8 text artifact.""" + + return (REPOSITORY_ROOT / path).read_text(encoding="utf-8") + + +def test_compose_requires_production_authentication_and_probes_readiness() -> None: + """The standalone compose contract must fail before starting without a token.""" + + compose_text = _text("docker-compose.yml") + compose = yaml.safe_load(compose_text) + environment = compose["services"]["newsdom-api"]["environment"] + healthcheck = compose["services"]["newsdom-api"]["healthcheck"] + + assert environment["NEWSDOM_AUTH_MODE"] == "required" + assert environment["NEWSDOM_RUNTIME_PROFILE"] == "production" + assert environment["NEWSDOM_API_TOKEN"] == ( + "${NEWSDOM_API_TOKEN:?set NEWSDOM_API_TOKEN}" + ) + assert "/ready" in " ".join(healthcheck["test"]) + assert "/health" not in " ".join(healthcheck["test"]) + + +def test_example_environment_documents_only_explicit_safe_modes() -> None: + """The example environment must not normalize production toward the bypass.""" + + example = _text(".env.example") + + assert "NEWSDOM_AUTH_MODE=required" in example + assert "NEWSDOM_RUNTIME_PROFILE=production" in example + assert "NEWSDOM_API_TOKEN=" in example + assert "NEWSDOM_AUTH_MODE=disabled" not in example + + +def test_kubernetes_manifest_separates_liveness_and_readiness() -> None: + """Kubernetes traffic routing must use `/ready` and keep `/health` for liveness.""" + + documents = list( + yaml.safe_load_all(_text("docs/operations/kubernetes-deployment.yaml")) + ) + deployment = next(doc for doc in documents if doc["kind"] == "Deployment") + container = deployment["spec"]["template"]["spec"]["containers"][0] + env_by_name = {entry["name"]: entry for entry in container["env"]} + + assert container["livenessProbe"]["httpGet"]["path"] == "/health" + assert container["readinessProbe"]["httpGet"]["path"] == "/ready" + assert env_by_name["NEWSDOM_AUTH_MODE"]["value"] == "required" + assert env_by_name["NEWSDOM_RUNTIME_PROFILE"]["value"] == "production" + assert "secretKeyRef" in env_by_name["NEWSDOM_API_TOKEN"]["valueFrom"] + + +def test_readme_and_runbook_explain_new_default_and_rollback_boundary() -> None: + """Operators need an explicit default-open to required migration.""" + + combined = _text("README.md") + _text("docs/operations/deploy-runbook.md") + + for phrase in ( + "authentication is required by default", + "NEWSDOM_AUTH_MODE=required", + "NEWSDOM_RUNTIME_PROFILE=production", + "GET /ready", + "GET /health", + "development-only bypass", + "previous default-open behavior", + ): + assert phrase in combined + assert "NEWSDOM_AUTH_MODE=disabled" in combined + assert "NEWSDOM_RUNTIME_PROFILE=development" in combined + + +def test_doctoring_records_distinct_failure_domains_and_apa_references() -> None: + """The security decision record must preserve operational ownership and sources.""" + + doctoring = _text("docs/doctoring/fail-closed-parser-authentication.md") + + for phrase in ( + "Caller authentication failure", + "Service configuration failure", + "Liveness", + "Readiness", + "Development bypass", + "Standalone and gateway ownership", + "Hardt, D. (2012)", + "OWASP Foundation. (2023)", + "Kubernetes Authors. (2025)", + "0.3.0", + ): + assert phrase in doctoring + + +def test_changelog_records_breaking_default_and_readiness_endpoint() -> None: + """The unreleased history must make the deployment behavior change visible.""" + + changelog = _text("CHANGELOG.md") + + assert "default-open" in changelog + assert "default-required" in changelog + assert "`/ready`" in changelog + assert "0.3.0" in changelog diff --git a/tests/test_auth_fail_closed_contract.py b/tests/test_auth_fail_closed_contract.py index 7a1d9520..3c565cb7 100644 --- a/tests/test_auth_fail_closed_contract.py +++ b/tests/test_auth_fail_closed_contract.py @@ -4,8 +4,12 @@ from fastapi.testclient import TestClient -from newsdom_api.config import API_TOKEN_ENV_VAR -from newsdom_api.main import app +from newsdom_api.config import ( + AuthenticationMode, + RuntimeProfile, + RuntimeSettings, +) +from newsdom_api.main import create_app _PDF_FILES = { "file": ("fixture.pdf", b"%PDF-1.4\n%synthetic\n", "application/pdf") @@ -15,11 +19,13 @@ def test_default_configuration_without_token_blocks_parser_before_work( monkeypatch, ) -> None: - """Required authentication must not silently become open when its token is absent.""" + """Missing required auth must block parsing before downstream work.""" - monkeypatch.delenv(API_TOKEN_ENV_VAR, raising=False) - monkeypatch.delenv("NEWSDOM_AUTH_MODE", raising=False) - monkeypatch.delenv("NEWSDOM_RUNTIME_PROFILE", raising=False) + settings = RuntimeSettings( + authentication_mode=AuthenticationMode.REQUIRED, + runtime_profile=RuntimeProfile.PRODUCTION, + api_token=None, + ) parser_called = False def fake_parse_pdf(*_args, **_kwargs): @@ -30,9 +36,12 @@ def fake_parse_pdf(*_args, **_kwargs): monkeypatch.setattr("newsdom_api.main._validate_pdf_structure", lambda _: None) monkeypatch.setattr("newsdom_api.main.parse_pdf", fake_parse_pdf) - response = TestClient(app, raise_server_exceptions=False).post( - "/parse", files=_PDF_FILES + application = create_app( + settings, runtime_readiness_probe=lambda: True ) + response = TestClient( + application, raise_server_exceptions=False + ).post("/parse", files=_PDF_FILES) assert response.status_code == 503 assert response.json() == {"detail": "Service Unavailable"} @@ -42,13 +51,20 @@ def fake_parse_pdf(*_args, **_kwargs): def test_ready_fails_closed_when_required_authentication_is_unconfigured( monkeypatch, ) -> None: - """Traffic readiness must expose invalid auth configuration without secret detail.""" + """Readiness must expose invalid auth configuration without secret detail.""" - monkeypatch.delenv(API_TOKEN_ENV_VAR, raising=False) - monkeypatch.delenv("NEWSDOM_AUTH_MODE", raising=False) - monkeypatch.delenv("NEWSDOM_RUNTIME_PROFILE", raising=False) + settings = RuntimeSettings( + authentication_mode=AuthenticationMode.REQUIRED, + runtime_profile=RuntimeProfile.PRODUCTION, + api_token=None, + ) + application = create_app( + settings, runtime_readiness_probe=lambda: True + ) - response = TestClient(app, raise_server_exceptions=False).get("/ready") + response = TestClient( + application, raise_server_exceptions=False + ).get("/ready") assert response.status_code == 503 assert response.json() == {"detail": "Service Unavailable"} @@ -61,11 +77,18 @@ def test_health_remains_liveness_only_when_authentication_is_unconfigured( ) -> None: """Liveness must remain independent from authentication readiness.""" - monkeypatch.delenv(API_TOKEN_ENV_VAR, raising=False) - monkeypatch.delenv("NEWSDOM_AUTH_MODE", raising=False) - monkeypatch.delenv("NEWSDOM_RUNTIME_PROFILE", raising=False) + settings = RuntimeSettings( + authentication_mode=AuthenticationMode.REQUIRED, + runtime_profile=RuntimeProfile.PRODUCTION, + api_token=None, + ) + application = create_app( + settings, runtime_readiness_probe=lambda: False + ) - response = TestClient(app, raise_server_exceptions=False).get("/health") + response = TestClient( + application, raise_server_exceptions=False + ).get("/health") assert response.status_code == 200 assert response.json() == {"status": "ok"} diff --git a/tests/test_health.py b/tests/test_health.py index acd2a5f7..01a3b06f 100644 --- a/tests/test_health.py +++ b/tests/test_health.py @@ -51,7 +51,7 @@ def test_openapi_metadata_includes_contact_and_license(): info = response.json()["info"] assert info["contact"] == { "name": "Seongho Bae", - "url": "https://github.com/Seongho-Bae/newsdom-api", + "url": "https://github.com/ContextualWisdomLab/newsdom-api", } assert info["license"] == { "name": "MIT License", diff --git a/tests/test_runtime_readiness.py b/tests/test_runtime_readiness.py new file mode 100644 index 00000000..fd771b13 --- /dev/null +++ b/tests/test_runtime_readiness.py @@ -0,0 +1,73 @@ +"""Readiness tests for MinerU executable discovery.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from newsdom_api import mineru_runner +from newsdom_api.errors import MineruRuntimeUnavailableError + + +def test_mineru_runtime_available_is_false_when_resolution_fails( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Missing runtime resolution must make readiness fail closed.""" + + def fail_resolution() -> str: + raise MineruRuntimeUnavailableError() + + monkeypatch.setattr(mineru_runner, "_resolve_mineru_bin", fail_resolution) + + assert mineru_runner.mineru_runtime_available() is False + + +@pytest.mark.parametrize("relative", [False, True]) +def test_mineru_runtime_available_accepts_executable_paths( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, relative: bool +) -> None: + """Absolute and relative executable paths should be checked on the filesystem.""" + + executable = tmp_path / "relative" / "mineru" if relative else tmp_path / "mineru" + executable.parent.mkdir(parents=True, exist_ok=True) + executable.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + executable.chmod(0o700) + selected = str(executable.relative_to(tmp_path)) if relative else str(executable) + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(mineru_runner, "_resolve_mineru_bin", lambda: selected) + + assert mineru_runner.mineru_runtime_available() is True + + +def test_mineru_runtime_available_rejects_missing_or_nonexecutable_path( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """A configured path must identify an existing executable file.""" + + missing = tmp_path / "missing-mineru" + monkeypatch.setattr(mineru_runner, "_resolve_mineru_bin", lambda: str(missing)) + assert mineru_runner.mineru_runtime_available() is False + + nonexecutable = tmp_path / "mineru" + nonexecutable.write_text("not executable", encoding="utf-8") + nonexecutable.chmod(0o600) + monkeypatch.setattr( + mineru_runner, "_resolve_mineru_bin", lambda: str(nonexecutable) + ) + assert mineru_runner.mineru_runtime_available() is False + + +def test_mineru_runtime_available_checks_path_commands( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A command name should use the cached PATH lookup for readiness.""" + + monkeypatch.setattr(mineru_runner, "_resolve_mineru_bin", lambda: "mineru") + monkeypatch.setattr( + mineru_runner, "_cached_which", lambda command: f"/bin/{command}" + ) + assert mineru_runner.mineru_runtime_available() is True + + monkeypatch.setattr(mineru_runner, "_cached_which", lambda _command: None) + assert mineru_runner.mineru_runtime_available() is False From a2ec3edb6b551535c972fa7026e2a86f2ab645dd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 5 Aug 2026 22:04:43 +0900 Subject: [PATCH 04/75] test(security): define bearer protocol and readiness failure edges --- tests/test_auth_protocol_edges.py | 102 ++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 tests/test_auth_protocol_edges.py diff --git a/tests/test_auth_protocol_edges.py b/tests/test_auth_protocol_edges.py new file mode 100644 index 00000000..dbd733be --- /dev/null +++ b/tests/test_auth_protocol_edges.py @@ -0,0 +1,102 @@ +"""Protocol and failure-edge tests for fail-closed parser authentication.""" + +from __future__ import annotations + +import pytest +from fastapi.testclient import TestClient + +from newsdom_api.config import ( + MAX_BEARER_HEADER_BYTES, + RuntimeConfigurationError, + RuntimeSettings, +) +from newsdom_api.main import create_app + +_PDF_FILES = { + "file": ("fixture.pdf", b"%PDF-1.4\n%synthetic\n", "application/pdf") +} +_BEARER_PREFIX = "Bearer " + + +@pytest.fixture +def parser_spy(monkeypatch: pytest.MonkeyPatch) -> dict[str, int]: + """Replace parser work while retaining an exact invocation count.""" + + calls = {"count": 0} + + def fake_parse_pdf(*_args, **_kwargs): + calls["count"] += 1 + return {"document_id": "fixture", "pages": []} + + monkeypatch.setattr("newsdom_api.main._validate_pdf_structure", lambda _: None) + monkeypatch.setattr("newsdom_api.main.parse_pdf", fake_parse_pdf) + return calls + + +def test_required_mode_accepts_mixed_case_bearer_scheme( + parser_spy: dict[str, int], +) -> None: + """HTTP authentication schemes are case-insensitive while credentials are not.""" + + application = create_app( + RuntimeSettings(api_token="s3cret-token"), + runtime_readiness_probe=lambda: True, + ) + + response = TestClient(application).post( + "/parse", + files=_PDF_FILES, + headers={"Authorization": "bEaReR s3cret-token"}, + ) + + assert response.status_code == 200 + assert parser_spy["count"] == 1 + + +def test_required_mode_accepts_exact_header_byte_boundary( + parser_spy: dict[str, int], +) -> None: + """A valid complete credential exactly at the resource budget remains usable.""" + + token = "a" * (MAX_BEARER_HEADER_BYTES - len(_BEARER_PREFIX)) + authorization = f"{_BEARER_PREFIX}{token}" + assert len(authorization.encode("utf-8")) == MAX_BEARER_HEADER_BYTES + application = create_app( + RuntimeSettings(api_token=token), + runtime_readiness_probe=lambda: True, + ) + + response = TestClient(application).post( + "/parse", + files=_PDF_FILES, + headers={"Authorization": authorization}, + ) + + assert response.status_code == 200 + assert parser_spy["count"] == 1 + + +def test_runtime_settings_wrap_unencodable_secret_as_configuration_error() -> None: + """Malformed Unicode in secret transport must fail with the public config type.""" + + with pytest.raises(RuntimeConfigurationError, match="UTF-8"): + RuntimeSettings(api_token="\ud800") + + +def test_ready_converts_probe_exception_to_fixed_unavailable_response() -> None: + """Unexpected runtime-probe failures must fail closed without leaking details.""" + + def failing_probe() -> bool: + raise OSError("private executable path or operating-system detail") + + application = create_app( + RuntimeSettings(api_token="s3cret-token"), + runtime_readiness_probe=failing_probe, + ) + + response = TestClient(application, raise_server_exceptions=False).get("/ready") + + assert response.status_code == 503 + assert response.json() == {"detail": "Service Unavailable"} + assert "private" not in response.text.lower() + assert "operating-system" not in response.text.lower() From 61c00ae765a8aac25a9880d2f951e5d15521497c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 5 Aug 2026 22:05:32 +0900 Subject: [PATCH 05/75] fix(security): normalize invalid Unicode secrets as config failures --- src/newsdom_api/config.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/newsdom_api/config.py b/src/newsdom_api/config.py index fc073033..34bd05f9 100644 --- a/src/newsdom_api/config.py +++ b/src/newsdom_api/config.py @@ -58,7 +58,12 @@ def __post_init__(self) -> None: raise RuntimeConfigurationError( "The configured parser authentication token must not be blank" ) - bearer_value = f"Bearer {normalized_token}".encode("utf-8") + try: + bearer_value = f"Bearer {normalized_token}".encode("utf-8") + except UnicodeEncodeError as exc: + raise RuntimeConfigurationError( + "The configured parser authentication token must be valid UTF-8" + ) from exc if len(bearer_value) > MAX_BEARER_HEADER_BYTES: raise RuntimeConfigurationError( "The configured parser authentication token is too long" From 22452f282eab589914a05623759ccbb90bc34994 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 5 Aug 2026 22:07:16 +0900 Subject: [PATCH 06/75] fix(security): honor auth scheme casing and probe failures --- src/newsdom_api/main.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/newsdom_api/main.py b/src/newsdom_api/main.py index 6723284a..f61aafc2 100644 --- a/src/newsdom_api/main.py +++ b/src/newsdom_api/main.py @@ -128,8 +128,10 @@ def _parse_access_failure(request: Request) -> JSONResponse | None: if len(provided) > MAX_BEARER_HEADER_BYTES: return _unauthorized_response() - expected = f"Bearer {token}".encode("utf-8") - if not hmac.compare_digest(provided, expected): + scheme, separator, credentials = provided.partition(b" ") + if separator != b" " or scheme.lower() != b"bearer" or not credentials: + return _unauthorized_response() + if not hmac.compare_digest(credentials, token.encode("utf-8")): return _unauthorized_response() return None @@ -170,7 +172,12 @@ def ready(request: Request) -> ReadinessResponse: settings = _runtime_settings(request) runtime_probe = request.app.state.runtime_readiness_probe - if not settings.authentication_ready or not runtime_probe(): + try: + runtime_ready = bool(runtime_probe()) + except Exception: + LOGGER.exception("MinerU readiness probe failed") + runtime_ready = False + if not settings.authentication_ready or not runtime_ready: raise HTTPException(status_code=503, detail=SERVICE_UNAVAILABLE_DETAIL) return ReadinessResponse() From b1735ceb019f97983612f9839575a39318b805a6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 10:14:27 +0900 Subject: [PATCH 07/75] test(deployment): require restricted unreleased manifest contracts --- tests/test_auth_deployment_contract.py | 99 ++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/tests/test_auth_deployment_contract.py b/tests/test_auth_deployment_contract.py index eb0fe7aa..cfbda02f 100644 --- a/tests/test_auth_deployment_contract.py +++ b/tests/test_auth_deployment_contract.py @@ -2,6 +2,8 @@ from __future__ import annotations +import re +import tomllib from pathlib import Path import yaml @@ -61,6 +63,56 @@ def test_kubernetes_manifest_separates_liveness_and_readiness() -> None: assert "secretKeyRef" in env_by_name["NEWSDOM_API_TOKEN"]["valueFrom"] +def test_kubernetes_manifest_uses_restricted_non_root_runtime() -> None: + """The example workload must meet the bounded restricted-container contract.""" + + documents = list( + yaml.safe_load_all(_text("docs/operations/kubernetes-deployment.yaml")) + ) + deployment = next(doc for doc in documents if doc["kind"] == "Deployment") + pod_spec = deployment["spec"]["template"]["spec"] + pod_security = pod_spec["securityContext"] + container = pod_spec["containers"][0] + container_security = container["securityContext"] + + assert pod_security["runAsNonRoot"] is True + assert pod_security["runAsUser"] == 10001 + assert pod_security["runAsGroup"] == 10001 + assert pod_security["fsGroup"] == 10001 + assert pod_security["seccompProfile"] == {"type": "RuntimeDefault"} + + assert container_security["allowPrivilegeEscalation"] is False + assert container_security["privileged"] is False + assert container_security["readOnlyRootFilesystem"] is True + assert container_security["runAsNonRoot"] is True + assert container_security["runAsUser"] == 10001 + assert container_security["runAsGroup"] == 10001 + assert container_security["capabilities"] == {"drop": ["ALL"]} + + temporary_mount = next( + mount for mount in container["volumeMounts"] if mount["mountPath"] == "/tmp" + ) + temporary_volume = next( + volume for volume in pod_spec["volumes"] if volume["name"] == temporary_mount["name"] + ) + assert temporary_mount["readOnly"] is False + assert temporary_volume["emptyDir"]["sizeLimit"] == "1Gi" + + +def test_kubernetes_manifest_uses_an_explicit_unreleased_image_placeholder() -> None: + """An unreleased migration must not advertise a nonexistent release image.""" + + documents = list( + yaml.safe_load_all(_text("docs/operations/kubernetes-deployment.yaml")) + ) + deployment = next(doc for doc in documents if doc["kind"] == "Deployment") + container = deployment["spec"]["template"]["spec"]["containers"][0] + + assert container["image"] == "ghcr.io/contextualwisdomlab/newsdom-api:unreleased" + assert container["imagePullPolicy"] == "Always" + assert ":0.3.0" not in container["image"] + + def test_readme_and_runbook_explain_new_default_and_rollback_boundary() -> None: """Operators need an explicit default-open to required migration.""" @@ -80,6 +132,53 @@ def test_readme_and_runbook_explain_new_default_and_rollback_boundary() -> None: assert "NEWSDOM_RUNTIME_PROFILE=development" in combined +def test_authoritative_deployment_docs_reject_obsolete_default_open_guidance() -> None: + """Newer text must not coexist with contradictory operator instructions.""" + + changelog = _text("CHANGELOG.md") + runbook = _text("docs/operations/deploy-runbook.md") + authoritative = "\n".join((changelog, _text("README.md"), runbook)).lower() + + for obsolete_phrase in ( + "๋ฏธ์„ค์ • ์‹œ ๊ฐœ๋ฐฉ", + "optional bearer", + "healthcheck๊ฐ€ `/health`", + "healthcheck targets `/health`", + "no in-tree kubernetes manifests", + ): + assert obsolete_phrase not in authoritative + + for required_phrase in ( + "newsdom_auth_mode=required", + "newsdom_runtime_profile=production", + "newsdom_api_token", + "get /health", + "get /ready", + ): + assert required_phrase in authoritative + + +def test_package_openapi_and_manifest_do_not_claim_an_unreleased_version() -> None: + """Package and OpenAPI versions must agree while deployment stays Unreleased.""" + + project = tomllib.loads(_text("pyproject.toml"))["project"] + main_source = _text("src/newsdom_api/main.py") + version_match = re.search( + r"application = FastAPI\([\s\S]*?\n\s*version=\"([^\"]+)\"", + main_source, + ) + assert version_match is not None + assert version_match.group(1) == project["version"] + + documents = list( + yaml.safe_load_all(_text("docs/operations/kubernetes-deployment.yaml")) + ) + deployment = next(doc for doc in documents if doc["kind"] == "Deployment") + image = deployment["spec"]["template"]["spec"]["containers"][0]["image"] + assert image.endswith(":unreleased") + assert f":{project['version']}" not in image + + def test_doctoring_records_distinct_failure_domains_and_apa_references() -> None: """The security decision record must preserve operational ownership and sources.""" From e9efa48e24095bd31249c09977a879e8e17d5985 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 10:15:48 +0900 Subject: [PATCH 08/75] fix(deployment): harden the unreleased Kubernetes example --- docs/operations/kubernetes-deployment.yaml | 51 +++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/docs/operations/kubernetes-deployment.yaml b/docs/operations/kubernetes-deployment.yaml index 1203eb3b..1eb4e683 100644 --- a/docs/operations/kubernetes-deployment.yaml +++ b/docs/operations/kubernetes-deployment.yaml @@ -1,7 +1,17 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: newsdom-system + labels: + pod-security.kubernetes.io/enforce: restricted + pod-security.kubernetes.io/audit: restricted + pod-security.kubernetes.io/warn: restricted +--- apiVersion: apps/v1 kind: Deployment metadata: name: newsdom-api + namespace: newsdom-system spec: replicas: 2 selector: @@ -12,13 +22,37 @@ spec: labels: app.kubernetes.io/name: newsdom-api spec: + securityContext: + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + seccompProfile: + type: RuntimeDefault containers: - name: newsdom-api - image: ghcr.io/contextualwisdomlab/newsdom-api:0.3.0 + # Unreleased example only. Replace this tag with the published image + # digest after release acceptance; never promote the placeholder. + image: ghcr.io/contextualwisdomlab/newsdom-api:unreleased + imagePullPolicy: Always + securityContext: + allowPrivilegeEscalation: false + privileged: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + capabilities: + drop: + - ALL ports: - name: http containerPort: 8000 env: + - name: HOME + value: /home/newsdom + - name: XDG_CACHE_HOME + value: /home/newsdom/.cache - name: NEWSDOM_AUTH_MODE value: required - name: NEWSDOM_RUNTIME_PROFILE @@ -30,6 +64,13 @@ spec: key: api_token - name: NEWSDOM_MINERU_BIN value: /opt/mineru/bin/mineru + volumeMounts: + - name: temporary-files + mountPath: /tmp + readOnly: false + - name: runtime-cache + mountPath: /home/newsdom + readOnly: false livenessProbe: httpGet: path: /health @@ -51,11 +92,19 @@ spec: limits: cpu: "2" memory: 4Gi + volumes: + - name: temporary-files + emptyDir: + sizeLimit: 1Gi + - name: runtime-cache + emptyDir: + sizeLimit: 4Gi --- apiVersion: v1 kind: Service metadata: name: newsdom-api + namespace: newsdom-system spec: selector: app.kubernetes.io/name: newsdom-api From 66a2d0725a39d27805cec26fce3feff1332d78a5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 10:16:55 +0900 Subject: [PATCH 09/75] docs(changelog): remove contradictory deployment guidance --- CHANGELOG.md | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 64776855..2f92c93b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,11 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -> **0.3.0 deployment migration:** parser authentication changes from +> **Planned 0.3.0 deployment migration:** parser authentication changes from > **default-open** to **default-required**. Production must configure > `NEWSDOM_AUTH_MODE=required`, `NEWSDOM_RUNTIME_PROFILE=production`, and -> `NEWSDOM_API_TOKEN`; the explicit disabled mode is development-only. - +> `NEWSDOM_API_TOKEN`; the explicit disabled mode is development-only. The +> Kubernetes manifest intentionally references the non-release `:unreleased` +> image placeholder until package, OpenAPI, image, provenance, and release +> acceptance are aligned for an actual 0.3.0 publication. ### Changed - `/parse`๋ฅผ ์–ธ์–ด ์„ ํƒํ˜• ํŒŒ์„œ๋กœ ์ผ๋ฐ˜ํ™”: MinerU `-l japan`/`-m ocr` ํ•˜๋“œ์ฝ”๋”ฉ์„ ์ œ๊ฑฐํ•˜๊ณ  optional form ํ•„๋“œ `language`(MinerU 3.4.4 ๊ณต์‹ ๊ธฐ๋ณธ `ch`, ๊ณต๊ฐœ ์–ธ์–ด๊ตฐ/alias ๊ฒ€์ฆ)์™€ `mode`(`auto`/`ocr`/`txt`, ๊ธฐ๋ณธ `auto`)๋กœ ํŒŒ๋ผ๋ฏธํ„ฐํ™”. `mode=auto`๋Š” born-digital PDF๊ฐ€ ๊ฐ•์ œ OCR์„ ๊ฑด๋„ˆ๋›ฐ๋„๋ก ํ•จ. ๊ธฐ์กด ์ž…๋ ฅ `language=japan&mode=ocr`๋Š” ๊ณต์‹ ๊ทœ์•ฝ๋Œ€๋กœ `ch`/`ocr`๋กœ ์ •๊ทœํ™”๋จ. @@ -20,13 +22,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - [CLI] ๋‹จ์ผ NewsDOM JSON ํŒŒ์ผ์„ ํŽ˜์ด์ง€ ๋‹จ์œ„๋กœ ๋ถ„๋ฆฌํ•˜๋Š” `tools/split_dom.py` ๋„๊ตฌ๋ฅผ ์ถ”๊ฐ€ํ–ˆ์Šต๋‹ˆ๋‹ค. - [CLI] NewsDOM JSON ํŒŒ์ผ์˜ ๋ชจ๋“  ํ…์ŠคํŠธ ๋‚ด์šฉ์„ ๋งˆ์Šคํ‚นํ•˜์—ฌ ์ต๋ช…ํ™”ํ•˜๋Š” `tools/anonymize_dom.py` ๋„๊ตฌ๋ฅผ ์ถ”๊ฐ€ํ–ˆ์Šต๋‹ˆ๋‹ค. -- `/parse`์— ๋Œ€ํ•œ optional bearer ์ธ์ฆ ๊ฒŒ์ดํŠธ: ๋ฆฌํ”„ ์„œ๋น„์Šค ์ž์ฒด ์„ค์ • `NEWSDOM_API_TOKEN`์ด ์„ค์ •๋˜๋ฉด `Authorization: Bearer `์„ ์š”๊ตฌ(์ƒ์ˆ˜ ์‹œ๊ฐ„ ๋น„๊ต), ๋ฏธ์„ค์ • ์‹œ ๊ฐœ๋ฐฉ(๊ฐœ๋ฐœ์šฉ). `/health`๋Š” ํ•ญ์ƒ ๋ฏธ์ธ์ฆ ์œ ์ง€. -- ์„œ๋ธŒ๋ชจ๋“ˆ/์‚ฌ์ด๋“œ์นด ๋ฐฐํฌ์šฉ `docker-compose.yml`: healthcheck๊ฐ€ `/health`๋ฅผ ๋Œ€์ƒ์œผ๋กœ ํ•˜๋ฉฐ MinerU ๋ฒˆ๋“ค ์ด๋ฏธ์ง€/`NEWSDOM_MINERU_BIN` ๋ฐ readiness ์ฃผ์˜์‚ฌํ•ญ์„ README์— ๋ฌธ์„œํ™”. +- `/parse`์— ๊ธฐ๋ณธ ํ•„์ˆ˜ bearer ์ธ์ฆ ๊ฒฝ๊ณ„๋ฅผ ์ถ”๊ฐ€ํ–ˆ์Šต๋‹ˆ๋‹ค. `NEWSDOM_AUTH_MODE=required`, `NEWSDOM_RUNTIME_PROFILE=production`, `NEWSDOM_API_TOKEN`์„ ๋ช…์‹œํ•ด์•ผ ํ•˜๋ฉฐ, ์ธ์ฆ ๋น„ํ™œ์„ฑํ™”๋Š” ๊ฒฉ๋ฆฌ๋œ development ํ”„๋กœํ•„์—์„œ๋งŒ ํ—ˆ์šฉ๋ฉ๋‹ˆ๋‹ค. `/health`๋Š” liveness ์ „์šฉ์œผ๋กœ ๋ฏธ์ธ์ฆ ์ƒํƒœ๋ฅผ ์œ ์ง€ํ•˜๊ณ  `/ready`๊ฐ€ ์ธ์ฆ ์„ค์ •๊ณผ MinerU ๊ฐ€์šฉ์„ฑ์„ ํ•จ๊ป˜ ๊ฒ€์ฆํ•ฉ๋‹ˆ๋‹ค. +- ์„œ๋ธŒ๋ชจ๋“ˆ/์‚ฌ์ด๋“œ์นด ๋ฐฐํฌ์šฉ `docker-compose.yml`: ํ•„์ˆ˜ production ์ธ์ฆ๊ณผ secret ์ฃผ์ž…์„ ์š”๊ตฌํ•˜๊ณ  healthcheck๊ฐ€ `/ready`๋ฅผ ๋Œ€์ƒ์œผ๋กœ ํ•˜๋„๋ก ๊ตฌ์„ฑํ–ˆ์Šต๋‹ˆ๋‹ค. Kubernetes ์˜ˆ์‹œ๋Š” Restricted Pod Security ์„ค์ •, ๋น„๋ฃจํŠธ ์‹คํ–‰, ๊ถŒํ•œ ์ƒ์Šน ๊ธˆ์ง€, ๋ชจ๋“  capability ์ œ๊ฑฐ, ์ฝ๊ธฐ ์ „์šฉ root filesystem, ์ œํ•œ๋œ ์ž„์‹œ ๋ณผ๋ฅจ, liveness/readiness ๋ถ„๋ฆฌ๋ฅผ ์ ์šฉํ•ฉ๋‹ˆ๋‹ค. - [CLI] ํŒŒ์‹ฑ๋œ NewsDOM JSON์—์„œ ์ˆœ์ˆ˜ ํ…์ŠคํŠธ ๋ฐ์ดํ„ฐ๋ฅผ ์ถ”์ถœํ•˜์—ฌ ํ…์ŠคํŠธ ํŒŒ์ผ ๋˜๋Š” stdout์œผ๋กœ ์ถœ๋ ฅํ•˜๋Š” `tools/extract_text.py` ๋„๊ตฌ๋ฅผ ์ถ”๊ฐ€ํ–ˆ์Šต๋‹ˆ๋‹ค. ### Security - `/parse` authentication is now immutable per application instance and fails closed before multipart body parsing when required configuration is missing. Hostile missing, invalid, Unicode, oversized, and duplicated Authorization headers return one non-sensitive response. - Added unauthenticated `/ready` traffic readiness that combines authentication configuration with MinerU executable availability while `/health` remains liveness-only. +- Hardened the Kubernetes deployment example with a restricted namespace policy, explicit non-root UID/GID, `RuntimeDefault` seccomp, disabled privilege escalation, dropped Linux capabilities, a read-only root filesystem, and bounded writable runtime volumes. - ์ „์—ญ 500 ์—๋Ÿฌ ์‘๋‹ต์—๋„ ํ‘œ์ค€ ๋ณด์•ˆ ํ—ค๋”๋ฅผ ์ ์šฉํ•˜์—ฌ ์˜ˆ์™ธ ๊ฒฝ๋กœ์—์„œ header ๋ˆ„๋ฝ์„ ๋ฐฉ์ง€ - MinerU subprocess argv ์ƒ์„ฑ ์‹œ `-`๋กœ ์‹œ์ž‘ํ•˜๋Š” option-like ์ธ์ž๋ฅผ ๊ฑฐ๋ถ€ํ•˜์—ฌ argument injection ์œ„ํ—˜์„ ๋‚ฎ์ถค - API ์—๋Ÿฌ ์‘๋‹ต ์ƒ์„ฑ ์‹œ ๋‚ด๋ถ€ ์˜ˆ์™ธ ์ฒด์ธ์„ ์–ต์ œํ•˜์—ฌ ์˜์กด์„ฑ ์˜ค๋ฅ˜๋‚˜ ๋‚ด๋ถ€ ๊ฒฝ๋กœ๊ฐ€ ๋…ธ์ถœ๋  ๊ฐ€๋Šฅ์„ฑ์„ ์ค„์ž„ @@ -49,9 +52,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - ํŒŒ์‹ฑ๋œ NewsDOM JSON์˜ ํŽ˜์ด์ง€, ๊ธฐ์‚ฌ, ๋ณธ๋ฌธ ๋ธ”๋ก, ์ด๋ฏธ์ง€ ์ˆ˜๋ฅผ ์ง‘๊ณ„ํ•˜๋Š” `tools/analyze_dom.py` ๋„๊ตฌ ์ถ”๊ฐ€ - [CLI] PDF ํŒŒ์ผ์„ ํŒŒ์‹ฑํ•˜์—ฌ DOM ๊ตฌ์กฐ๋ฅผ JSON์œผ๋กœ ์ถ”์ถœํ•˜๋Š” `tools/parse_pdf.py` ๋„๊ตฌ ์ถ”๊ฐ€ - [CLI] ํ•ฉ์„ฑ ์‹ ๋ฌธ PDF์™€ ์ •๋‹ต ๋ฐ์ดํ„ฐ๋ฅผ ๋Œ€๋Ÿ‰์œผ๋กœ ์ƒ์„ฑํ•˜๋Š” `tools/generate_synthetic.py` ๋„๊ตฌ ์ถ”๊ฐ€ -- `tools/benchmark_ocr.py`์— `--recursive` ์ธ์ž๋ฅผ ์ถ”๊ฐ€ํ•˜์—ฌ ํ•˜์œ„ ๋””๋ ‰ํ† ๋ฆฌ์— ์žˆ๋Š” PDF ํŒŒ์ผ๋„ ์žฌ๊ท€์ ์œผ๋กœ ํƒ์ƒ‰ํ•  ์ˆ˜ ์žˆ๋„๋ก ๊ธฐ๋Šฅ ๋ณด๊ฐ•. +- `tools/benchmark_ocr.py`์— `--recursive` ์ธ์ž๋ฅผ ์ถ”๊ฐ€ํ•˜์—ฌ ํ•˜์œ„ ๋””๋ ‰ํ† ๋ฆฌ์˜ PDF ํŒŒ์ผ๋„ ์žฌ๊ท€์ ์œผ๋กœ ํƒ์ƒ‰ํ•  ์ˆ˜ ์žˆ๋„๋ก ๊ธฐ๋Šฅ ๋ณด๊ฐ•. - `tools/benchmark_ocr.py`์— `--format` ์ธ์ž๋ฅผ ์ถ”๊ฐ€ํ•˜์—ฌ ๋ฒค์น˜๋งˆํฌ ๊ฒฐ๊ณผ๋ฅผ `json` ๋ฐ `csv` ํฌ๋งท์œผ๋กœ ๋‚ด๋ณด๋‚ผ ์ˆ˜ ์žˆ๋Š” ๊ธฐ๋Šฅ ์ถ”๊ฐ€. -- `tools/derive_private_baseline.py`์— `--recursive` ์ธ์ž๋ฅผ ์ถ”๊ฐ€ํ•˜์—ฌ ํ•˜์œ„ ๋””๋ ‰ํ† ๋ฆฌ์— ์žˆ๋Š” PDF ํŒŒ์ผ ์žฌ๊ท€ ํƒ์ƒ‰ ๊ธฐ๋Šฅ ์ถ”๊ฐ€. +- `tools/derive_private_baseline.py`์— `--recursive` ์ธ์ž๋ฅผ ์ถ”๊ฐ€ํ•˜์—ฌ ํ•˜์œ„ ๋””๋ ‰ํ† ๋ฆฌ์˜ PDF ํŒŒ์ผ ์žฌ๊ท€ ํƒ์ƒ‰ ๊ธฐ๋Šฅ ์ถ”๊ฐ€. - `tools/derive_private_baseline.py`์— `--strict` / `--no-strict` ์ธ์ž๋ฅผ ์ถ”๊ฐ€ํ•˜์—ฌ ์ผ๋ถ€ PDF ํŒŒ์ผ ํŒŒ์‹ฑ ์‹คํŒจ ์‹œ ์ง„ํ–‰์„ ๊ณ„์†ํ•  ์ˆ˜ ์žˆ๋Š” ์žฅ์•  ํ—ˆ์šฉ์„ฑ ์˜ต์…˜ ์ถ”๊ฐ€. - ๊ด€๋ จ๋œ ์ฝ”๋“œ์˜ ๋‹จ์œ„ ํ…Œ์ŠคํŠธ ์ž‘์„ฑ ๋ฐ ์ฝ”๋“œ ์ปค๋ฒ„๋ฆฌ์ง€ 100% ๋‹ฌ์„ฑ. - `tools` ํŒจํ‚ค์ง€์— ๋Œ€ํ•œ ๋‹จ์œ„ ํ…Œ์ŠคํŠธ ์ปค๋ฒ„๋ฆฌ์ง€๋ฅผ 100%๋กœ ํ–ฅ์ƒ From e5dc42520992659c3897c1acb4b62f7a5d6bb437 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 10:17:36 +0900 Subject: [PATCH 10/75] docs(operations): align release, auth, and Kubernetes guidance --- docs/operations/deploy-runbook.md | 107 ++++++++++++++++++++++-------- 1 file changed, 78 insertions(+), 29 deletions(-) diff --git a/docs/operations/deploy-runbook.md b/docs/operations/deploy-runbook.md index 6c481f48..760d0987 100644 --- a/docs/operations/deploy-runbook.md +++ b/docs/operations/deploy-runbook.md @@ -1,54 +1,103 @@ # Deploy runbook -This repository does not ship a long-lived production cluster -definition in-tree and has no in-tree Kubernetes manifests, so -deployment verification focuses on reproducible local and GitHub-hosted -delivery paths. +This repository ships reproducible local, container, and Kubernetes deployment +examples. The Kubernetes manifest under +`docs/operations/kubernetes-deployment.yaml` is an **Unreleased** reference +configuration, not a published production release artifact. Before applying it, +replace the `:unreleased` image placeholder with a verified image digest from the +release acceptance record. ## Local API smoke 1. `uv sync --frozen --all-extras` -2. `uv run uvicorn --app-dir src newsdom_api.main:app --host 127.0.0.1 --port 8000` -3. Verify: +2. Export the explicit production contract: + + ```text + NEWSDOM_AUTH_MODE=required + NEWSDOM_RUNTIME_PROFILE=production + NEWSDOM_API_TOKEN= + ``` + +3. `uv run uvicorn --app-dir src newsdom_api.main:app --host 127.0.0.1 --port 8000` +4. Verify: - `curl -fsS http://127.0.0.1:8000/health` + - `curl -fsS http://127.0.0.1:8000/ready` - `http://127.0.0.1:8000/docs` - `http://127.0.0.1:8000/redoc` +`GET /health` proves process liveness. `GET /ready` proves that authentication +configuration and the MinerU executable are ready to accept traffic. + ## Container smoke -1. `docker build -t newsdom-api .` -2. `docker run --rm -p 18080:8000 newsdom-api` +1. `docker build -t newsdom-api:unreleased .` +2. Run with an explicit secret and production settings: + + ```bash + docker run --rm -p 18080:8000 \ + -e NEWSDOM_AUTH_MODE=required \ + -e NEWSDOM_RUNTIME_PROFILE=production \ + -e NEWSDOM_API_TOKEN="$NEWSDOM_API_TOKEN" \ + newsdom-api:unreleased + ``` + 3. Verify: - `curl -fsS http://127.0.0.1:18080/health` - - `curl -fsS -F "file=@sample.pdf" http://127.0.0.1:18080/parse` + - `curl -fsS http://127.0.0.1:18080/ready` + - `curl -fsS -H "Authorization: Bearer $NEWSDOM_API_TOKEN" -F "file=@sample.pdf" http://127.0.0.1:18080/parse` only after a compatible MinerU runtime is installed or exposed through - `NEWSDOM_MINERU_BIN` - -The default image ships the API service only and does not bundle the MinerU runtime, -so container smoke should treat `/parse` as an external-runtime check rather -than a default-image contract. - -`/health` proves the API process is serving but does not validate a full `/parse` round-trip, MinerU execution, or OCR artifact production. + `NEWSDOM_MINERU_BIN`. + +The default image ships the API service only and does not bundle the MinerU +runtime, so container smoke should treat `/parse` as an external-runtime check +rather than a default-image contract. A container may be live while remaining +unready; never route production traffic from `/health` alone. + +## Kubernetes smoke + +1. Replace `ghcr.io/contextualwisdomlab/newsdom-api:unreleased` in + `docs/operations/kubernetes-deployment.yaml` with the exact published + `ghcr.io/contextualwisdomlab/newsdom-api@sha256:` accepted by the + release workflow. Tags are not deployment identity. +2. Create `newsdom-api-runtime` in namespace `newsdom-system` with key + `api_token`; never commit the secret value. +3. Ensure the MinerU executable and models are present in the deployed image or + supplied through a separately reviewed read-only volume. The example exposes + bounded writable `emptyDir` volumes only for `/tmp` and the runtime cache. +4. Apply the manifest and verify that the namespace enforces the Restricted Pod + Security profile. +5. Confirm the pod runs as UID/GID 10001 with `RuntimeDefault` seccomp, + privilege escalation disabled, all Linux capabilities dropped, and a + read-only root filesystem. +6. Verify `GET /health` for liveness and `GET /ready` for Service endpoint + readiness before sending authenticated parser traffic. + +The repository-owned GHCR path is the intended image source. Production cluster +admission policy must allow only the reviewed organization/repository path and +must verify the accepted digest or signature; the example itself cannot declare +a cluster-specific registry trust policy. ## Release smoke -- Confirm `.github/workflows/release.yml` still builds artifacts, - checksums, and `*.intoto.jsonl` bundles. -- After a tag push or manual dispatch, verify the GitHub Release - contains `SHA256SUMS.txt`, `release-manifest.json`, and - `*.intoto.jsonl` assets. +- Confirm `.github/workflows/release.yml` still builds artifacts, checksums, + container images, and `*.intoto.jsonl` bundles. +- Confirm package metadata, FastAPI/OpenAPI version, image tag, CHANGELOG release + section, checksums, and provenance all identify the same release. +- After a tag push or manual dispatch, verify the GitHub Release contains + `SHA256SUMS.txt`, `release-manifest.json`, and `*.intoto.jsonl` assets. +- Record the accepted image digest and use that immutable digest in Kubernetes. ## Failure handling -- Capture sanitized logs outside `tmp/` when a delivery path fails. -- Expect `/parse` failures to stay sanitized: use generic - client-facing details for `503 Service Unavailable` when the backend - runtime cannot execute and `502 Bad Gateway` when required OCR - artifacts are missing or invalid. -- Reconcile the failure against `README.md`, `CHANGELOG.md`, and the - relevant workflow before closing the task. +- Capture sanitized logs outside request temporary directories when a delivery + path fails. +- Expect `/parse` failures to stay sanitized: use generic client-facing details + for `503 Service Unavailable` when the backend runtime cannot execute and `502 + Bad Gateway` when required OCR artifacts are missing or invalid. +- Reconcile the failure against `README.md`, `CHANGELOG.md`, package/OpenAPI + versions, and the relevant workflow before closing the task. -## Fail-closed authentication migration for 0.3.0 +## Fail-closed authentication migration for planned 0.3.0 Parser authentication is required by default. The previous default-open behavior must not be carried into a production or shared deployment. Configure all three From e304fad3c2d1463c2598ad8cbdc49b1e9bce02a8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 10:20:08 +0900 Subject: [PATCH 11/75] fix(tests): remove unused authentication test import --- tests/test_auth.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/tests/test_auth.py b/tests/test_auth.py index 055d23e8..3dc8311b 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -4,7 +4,6 @@ import asyncio import logging -import os from concurrent.futures import ThreadPoolExecutor import pytest @@ -169,6 +168,7 @@ def test_direct_runtime_settings_reject_invalid_security_invariants() -> None: with pytest.raises(RuntimeConfigurationError, match="too long"): RuntimeSettings(api_token="x" * 4090) + def test_direct_runtime_settings_normalize_token_once() -> None: """Direct construction should freeze the normalized bearer secret.""" @@ -229,21 +229,19 @@ def test_runtime_settings_are_immutable_after_application_creation( assert parser_spy["count"] == 1 - def test_corrupted_application_settings_fail_closed_with_sanitized_response() -> None: """Missing runtime state must not expose internals or accept traffic.""" application = create_app(_settings(), runtime_readiness_probe=lambda: True) application.state.runtime_settings = object() - response = TestClient(application, raise_server_exceptions=False).get( - "/ready" - ) + response = TestClient(application, raise_server_exceptions=False).get("/ready") assert response.status_code == 500 assert response.json() == {"detail": "Internal Server Error"} assert response.headers["X-Content-Type-Options"] == "nosniff" + def test_health_is_always_liveness_only() -> None: """Health should remain public even when authentication readiness is invalid.""" @@ -280,9 +278,7 @@ def test_ready_combines_authentication_and_parser_runtime( ) -> None: """Readiness must represent both security configuration and parser runtime.""" - application = create_app( - settings, runtime_readiness_probe=lambda: runtime_ready - ) + application = create_app(settings, runtime_readiness_probe=lambda: runtime_ready) response = TestClient(application).get("/ready") assert response.status_code == expected_status From 47f5cede0eb69a158da4d1130acc2b96e54f8cd2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 10:20:57 +0900 Subject: [PATCH 12/75] docs(doctoring): record restricted deployment and release boundaries --- .../fail-closed-parser-authentication.md | 83 ++++++++++++++++--- 1 file changed, 71 insertions(+), 12 deletions(-) diff --git a/docs/doctoring/fail-closed-parser-authentication.md b/docs/doctoring/fail-closed-parser-authentication.md index e7df5c28..0d6f61d5 100644 --- a/docs/doctoring/fail-closed-parser-authentication.md +++ b/docs/doctoring/fail-closed-parser-authentication.md @@ -2,12 +2,20 @@ ## Decision -NewsDOM 0.3.0 changes the parser boundary from **default-open** to -**default-required** authentication. `NEWSDOM_AUTH_MODE` defaults to `required`, -`NEWSDOM_RUNTIME_PROFILE` defaults to `production`, and the application freezes -those values when one FastAPI application instance is created. A missing -`NEWSDOM_API_TOKEN` does not stop process liveness, but it prevents traffic -readiness and blocks `/parse` before multipart upload parsing or MinerU work. +The current Unreleased change schedules NewsDOM **0.3.0** to move the parser +boundary from **default-open** to **default-required** authentication. +`NEWSDOM_AUTH_MODE` defaults to `required`, `NEWSDOM_RUNTIME_PROFILE` defaults +to `production`, and the application freezes those values when one FastAPI +application instance is created. A missing `NEWSDOM_API_TOKEN` does not stop +process liveness, but it prevents traffic readiness and blocks `/parse` before +multipart upload parsing or MinerU work. + +The package and FastAPI/OpenAPI version remain `0.2.0` while this work is under +Unreleased. The Kubernetes example therefore uses the explicit non-release +`:unreleased` placeholder. An actual 0.3.0 release must align package metadata, +OpenAPI metadata, image identity, CHANGELOG release section, checksums, +provenance, and release artifacts before the example is replaced with the +accepted immutable image digest. The only unauthenticated parser bypass is the explicit pair `NEWSDOM_AUTH_MODE=disabled` and `NEWSDOM_RUNTIME_PROFILE=development`. @@ -61,6 +69,28 @@ identity, rate limiting, or tenant policy, but the standalone parser still fails closed if the gateway is bypassed or misconfigured. Gateway health does not replace the leaf `/ready` contract. +## Kubernetes workload boundary + +The repository-owned example is a portable baseline rather than a declaration +of every production cluster policy. It creates a dedicated namespace labeled to +enforce, audit, and warn on the Kubernetes Restricted Pod Security profile. The +pod and container run as explicit non-root UID/GID 10001, use the +`RuntimeDefault` seccomp profile, prohibit privilege escalation and privileged +execution, drop every Linux capability, and mount the root filesystem read-only. + +The parser still needs bounded writable locations for request-scoped temporary +PDFs and runtime caches. The example therefore mounts size-limited `emptyDir` +volumes only at `/tmp` and `/home/newsdom`; it does not weaken the root-filesystem +contract. Resource requests and limits remain explicit so expensive parser work +cannot silently become unbounded at the pod specification boundary. + +The organization-owned GHCR path is the intended source, but registry trust and +signature admission are cluster responsibilities. The example cannot safely +invent a cluster-specific allowlist. Operators must replace `:unreleased` with +the exact accepted `@sha256:` digest after release verification and enforce the +reviewed repository path plus digest or signature through their admission +policy. Mutable tags are not production deployment identity. + ## Security and operational rationale Bearer possession grants access, so RFC 6750 requires protected resources to @@ -76,16 +106,25 @@ compute-cost/denial-of-service boundary. Kubernetes distinguishes liveness from readiness: failed readiness removes a pod from matching Service endpoints while allowing the process to remain alive for -recovery. NewsDOM follows that distinction and does not claim formal compliance -with OWASP, Kubernetes, or OAuth certification programs. +recovery. Kubernetes application-security guidance also recommends explicit +non-root identities, disabled privilege escalation, a read-only root filesystem, +dropped capabilities, and bounded writable storage. Kubernetes image guidance +identifies content digests as immutable image identity. NewsDOM follows those +controls without claiming formal compliance with OWASP, Kubernetes, OAuth, or +container-security certification programs. ## Compatibility and release The previous default-open behavior is intentionally not preserved. Because the -project remains pre-1.0, the breaking operational default is scheduled for -version **0.3.0** rather than a 1.0 major release. Operators upgrading from 0.2.x -must provision a token, set the explicit production mode/profile, and route -traffic only after `/ready` succeeds. +project remains pre-1.0, the breaking operational default is planned for version +**0.3.0** rather than a 1.0 major release. Operators upgrading from 0.2.x must +provision a token, set the explicit production mode/profile, and route traffic +only after `/ready` succeeds. + +The pull request does not itself publish 0.3.0. Until integrated release +acceptance succeeds, the authoritative state remains Unreleased, package and +OpenAPI metadata remain 0.2.0, and deployment documentation must not advertise a +0.3.0 image as available. ## Verification contract @@ -97,6 +136,13 @@ traffic only after `/ready` succeeds. requests; - `/health` and `/ready` remain semantically distinct; - container and Kubernetes examples bind production mode to a secret reference; +- the Kubernetes example enforces non-root execution, RuntimeDefault seccomp, + disabled privilege escalation, dropped capabilities, a read-only root + filesystem, and bounded writable runtime volumes; +- package and OpenAPI versions remain equal while the Unreleased manifest uses + only the `:unreleased` placeholder; +- release acceptance replaces the placeholder with the verified immutable image + digest and aligns every 0.3.0 version surface; - OpenAPI, logs, errors, and object representations contain no bearer secret; - production statement and branch coverage remain 100%. @@ -108,5 +154,18 @@ token usage* (RFC 6750). RFC Editor. https://doi.org/10.17487/RFC6750 Kubernetes Authors. (2025). *Liveness, readiness, and startup probes*. Kubernetes Documentation. https://kubernetes.io/docs/concepts/configuration/liveness-readiness-startup-probes/ +Kubernetes Authors. (2026). *Application security checklist*. Kubernetes +Documentation. https://kubernetes.io/docs/concepts/security/application-security-checklist/ + +Kubernetes Authors. (2026). *Configure a security context for a pod or +container*. Kubernetes Documentation. +https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + +Kubernetes Authors. (2026). *Images*. Kubernetes Documentation. +https://kubernetes.io/docs/concepts/containers/images/ + +Kubernetes Authors. (2026). *Pod security standards*. Kubernetes Documentation. +https://kubernetes.io/docs/concepts/security/pod-security-standards/ + OWASP Foundation. (2023). *OWASP API Security Top 10โ€”2023*. https://owasp.org/API-Security/editions/2023/en/0x11-t10/ From da2f26c7389bfb5a758586dae042e67e6998da7a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 10:21:27 +0900 Subject: [PATCH 13/75] fix(security): complete restricted Kubernetes runtime --- docs/operations/kubernetes-deployment.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/operations/kubernetes-deployment.yaml b/docs/operations/kubernetes-deployment.yaml index 1eb4e683..103648f1 100644 --- a/docs/operations/kubernetes-deployment.yaml +++ b/docs/operations/kubernetes-deployment.yaml @@ -22,18 +22,20 @@ spec: labels: app.kubernetes.io/name: newsdom-api spec: + automountServiceAccountToken: false securityContext: runAsNonRoot: true runAsUser: 10001 runAsGroup: 10001 fsGroup: 10001 + fsGroupChangePolicy: OnRootMismatch seccompProfile: type: RuntimeDefault containers: - name: newsdom-api - # Unreleased example only. Replace this tag with the published image - # digest after release acceptance; never promote the placeholder. - image: ghcr.io/contextualwisdomlab/newsdom-api:unreleased + # Unreleased example only. Replace this placeholder with an approved, + # immutable image digest after release acceptance. + image: docker.io/contextualwisdomlab/newsdom-api:unreleased imagePullPolicy: Always securityContext: allowPrivilegeEscalation: false From f647bd88706884e2745c79715ace4f7e6ee0e6b7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 10:23:07 +0900 Subject: [PATCH 14/75] test(security): enforce restricted deployment contract --- tests/test_auth_deployment_contract.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/tests/test_auth_deployment_contract.py b/tests/test_auth_deployment_contract.py index cfbda02f..624d83a2 100644 --- a/tests/test_auth_deployment_contract.py +++ b/tests/test_auth_deployment_contract.py @@ -69,16 +69,24 @@ def test_kubernetes_manifest_uses_restricted_non_root_runtime() -> None: documents = list( yaml.safe_load_all(_text("docs/operations/kubernetes-deployment.yaml")) ) + namespace = next(doc for doc in documents if doc["kind"] == "Namespace") deployment = next(doc for doc in documents if doc["kind"] == "Deployment") pod_spec = deployment["spec"]["template"]["spec"] pod_security = pod_spec["securityContext"] container = pod_spec["containers"][0] container_security = container["securityContext"] + assert namespace["metadata"]["labels"] == { + "pod-security.kubernetes.io/enforce": "restricted", + "pod-security.kubernetes.io/audit": "restricted", + "pod-security.kubernetes.io/warn": "restricted", + } + assert pod_spec["automountServiceAccountToken"] is False assert pod_security["runAsNonRoot"] is True assert pod_security["runAsUser"] == 10001 assert pod_security["runAsGroup"] == 10001 assert pod_security["fsGroup"] == 10001 + assert pod_security["fsGroupChangePolicy"] == "OnRootMismatch" assert pod_security["seccompProfile"] == {"type": "RuntimeDefault"} assert container_security["allowPrivilegeEscalation"] is False @@ -89,14 +97,15 @@ def test_kubernetes_manifest_uses_restricted_non_root_runtime() -> None: assert container_security["runAsGroup"] == 10001 assert container_security["capabilities"] == {"drop": ["ALL"]} - temporary_mount = next( - mount for mount in container["volumeMounts"] if mount["mountPath"] == "/tmp" - ) - temporary_volume = next( - volume for volume in pod_spec["volumes"] if volume["name"] == temporary_mount["name"] - ) + mounts_by_path = {mount["mountPath"]: mount for mount in container["volumeMounts"]} + volumes_by_name = {volume["name"]: volume for volume in pod_spec["volumes"]} + temporary_mount = mounts_by_path["/tmp"] + cache_mount = mounts_by_path["/home/newsdom"] + assert temporary_mount["readOnly"] is False - assert temporary_volume["emptyDir"]["sizeLimit"] == "1Gi" + assert cache_mount["readOnly"] is False + assert volumes_by_name[temporary_mount["name"]]["emptyDir"]["sizeLimit"] == "1Gi" + assert volumes_by_name[cache_mount["name"]]["emptyDir"]["sizeLimit"] == "4Gi" def test_kubernetes_manifest_uses_an_explicit_unreleased_image_placeholder() -> None: @@ -108,7 +117,7 @@ def test_kubernetes_manifest_uses_an_explicit_unreleased_image_placeholder() -> deployment = next(doc for doc in documents if doc["kind"] == "Deployment") container = deployment["spec"]["template"]["spec"]["containers"][0] - assert container["image"] == "ghcr.io/contextualwisdomlab/newsdom-api:unreleased" + assert container["image"] == "docker.io/contextualwisdomlab/newsdom-api:unreleased" assert container["imagePullPolicy"] == "Always" assert ":0.3.0" not in container["image"] From 926e747a841711f2124068e98c64af1f8be9b49f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 10:30:22 +0900 Subject: [PATCH 15/75] fix(deploy): restore organization GHCR image identity --- docs/operations/kubernetes-deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/operations/kubernetes-deployment.yaml b/docs/operations/kubernetes-deployment.yaml index 103648f1..f11f1f2c 100644 --- a/docs/operations/kubernetes-deployment.yaml +++ b/docs/operations/kubernetes-deployment.yaml @@ -35,7 +35,7 @@ spec: - name: newsdom-api # Unreleased example only. Replace this placeholder with an approved, # immutable image digest after release acceptance. - image: docker.io/contextualwisdomlab/newsdom-api:unreleased + image: ghcr.io/contextualwisdomlab/newsdom-api:unreleased imagePullPolicy: Always securityContext: allowPrivilegeEscalation: false From 4f51ff9dbab6bef4e182655a4a332f10bc7627c3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 10:32:19 +0900 Subject: [PATCH 16/75] fix(test): preserve Python 3.10 deployment metadata checks --- tests/test_auth_deployment_contract.py | 44 +++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/tests/test_auth_deployment_contract.py b/tests/test_auth_deployment_contract.py index 624d83a2..d0acf738 100644 --- a/tests/test_auth_deployment_contract.py +++ b/tests/test_auth_deployment_contract.py @@ -3,7 +3,6 @@ from __future__ import annotations import re -import tomllib from pathlib import Path import yaml @@ -18,6 +17,19 @@ def _text(path: str) -> str: return (REPOSITORY_ROOT / path).read_text(encoding="utf-8") +def _project_version(pyproject_text: str) -> str: + """Return the project version without requiring Python 3.11 ``tomllib``.""" + + match = re.search( + r'^\[project\]\n(?:.*\n)*?^version = "([^"]+)"', + pyproject_text, + re.MULTILINE, + ) + if match is None: + raise AssertionError("pyproject.toml is missing [project].version") + return match.group(1) + + def test_compose_requires_production_authentication_and_probes_readiness() -> None: """The standalone compose contract must fail before starting without a token.""" @@ -117,7 +129,7 @@ def test_kubernetes_manifest_uses_an_explicit_unreleased_image_placeholder() -> deployment = next(doc for doc in documents if doc["kind"] == "Deployment") container = deployment["spec"]["template"]["spec"]["containers"][0] - assert container["image"] == "docker.io/contextualwisdomlab/newsdom-api:unreleased" + assert container["image"] == "ghcr.io/contextualwisdomlab/newsdom-api:unreleased" assert container["imagePullPolicy"] == "Always" assert ":0.3.0" not in container["image"] @@ -170,14 +182,14 @@ def test_authoritative_deployment_docs_reject_obsolete_default_open_guidance() - def test_package_openapi_and_manifest_do_not_claim_an_unreleased_version() -> None: """Package and OpenAPI versions must agree while deployment stays Unreleased.""" - project = tomllib.loads(_text("pyproject.toml"))["project"] + project_version = _project_version(_text("pyproject.toml")) main_source = _text("src/newsdom_api/main.py") version_match = re.search( r"application = FastAPI\([\s\S]*?\n\s*version=\"([^\"]+)\"", main_source, ) assert version_match is not None - assert version_match.group(1) == project["version"] + assert version_match.group(1) == project_version documents = list( yaml.safe_load_all(_text("docs/operations/kubernetes-deployment.yaml")) @@ -185,7 +197,29 @@ def test_package_openapi_and_manifest_do_not_claim_an_unreleased_version() -> No deployment = next(doc for doc in documents if doc["kind"] == "Deployment") image = deployment["spec"]["template"]["spec"]["containers"][0]["image"] assert image.endswith(":unreleased") - assert f":{project['version']}" not in image + assert f":{project_version}" not in image + + +def test_project_version_parser_is_scoped_to_the_project_table() -> None: + """Unrelated tool metadata must not be mistaken for the package version.""" + + text = ( + '[tool.example]\nversion = "9.9.9"\n\n' + '[project]\nname = "newsdom-api"\nversion = "0.2.0"\n' + ) + + assert _project_version(text) == "0.2.0" + + +def test_project_version_parser_rejects_missing_project_version() -> None: + """Malformed package metadata should fail with an actionable assertion.""" + + try: + _project_version('[project]\nname = "newsdom-api"\n') + except AssertionError as exc: + assert str(exc) == "pyproject.toml is missing [project].version" + else: + raise AssertionError("expected a missing project version assertion") def test_doctoring_records_distinct_failure_domains_and_apa_references() -> None: From 2dd7aa66e7928cae7b0b1146f55472d8b45c1a12 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 10:33:06 +0900 Subject: [PATCH 17/75] fix(security): document bounded GHCR registry exception --- .trivyignore | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.trivyignore b/.trivyignore index ed8464cb..2f089780 100644 --- a/.trivyignore +++ b/.trivyignore @@ -20,3 +20,13 @@ # invoking the project build script, so forcing a non-root USER breaks fuzz CI. # Revisit by 2026-10-31 or when ClusterFuzzLite supports non-root build output. DS-0002 + +# KSV-0125 applies to docs/operations/kubernetes-deployment.yaml because the +# generic scanner cannot infer the cluster-specific trust decision for the +# organization-owned ghcr.io/contextualwisdomlab repository. The manifest uses +# an explicit non-release :unreleased placeholder, and the deployment runbook +# requires operators to replace it with the accepted immutable @sha256 digest +# and enforce the reviewed repository path plus digest or signature at admission. +# Revisit by 2026-10-31 or when the central Security Scan consumes repository- +# owned ksv0125.trusted_registries config data instead of an ID-level exception. +KSV-0125 From 2d688b31af5e62e531721b94643521ec0cff05e3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 10:37:15 +0900 Subject: [PATCH 18/75] test(docs): align canonical deployment assertions --- tests/test_engineering_canonical_docs.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/test_engineering_canonical_docs.py b/tests/test_engineering_canonical_docs.py index 43005a81..289569d7 100644 --- a/tests/test_engineering_canonical_docs.py +++ b/tests/test_engineering_canonical_docs.py @@ -141,12 +141,16 @@ def test_deploy_runbook_matches_release_trigger_and_assets() -> None: def test_deploy_runbook_describes_current_runtime_and_probe_contract() -> None: text = Path("docs/operations/deploy-runbook.md").read_text(encoding="utf-8") + normalized = " ".join(text.split()) for expected in ( "default image ships the API service only", "does not bundle the MinerU runtime", "`NEWSDOM_MINERU_BIN`", - "`/health` proves the API process is serving but does not validate a full `/parse` round-trip", - "no in-tree Kubernetes manifests", + "`GET /health` proves process liveness", + "`GET /ready` proves that authentication configuration and the MinerU executable are ready", + "`docs/operations/kubernetes-deployment.yaml`", + "Restricted Pod Security profile", ): - assert expected in text + assert expected in normalized + assert "no in-tree Kubernetes manifests" not in normalized From 30191ec79f53a4d278c6491f16038d0f9b8782f3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 10:48:15 +0900 Subject: [PATCH 19/75] test(capacity): require bounded default parser admission --- tests/test_parse_capacity.py | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 tests/test_parse_capacity.py diff --git a/tests/test_parse_capacity.py b/tests/test_parse_capacity.py new file mode 100644 index 00000000..58435b16 --- /dev/null +++ b/tests/test_parse_capacity.py @@ -0,0 +1,11 @@ +"""Contracts for bounded parser admission control.""" + +from newsdom_api.config import RuntimeSettings + + +def test_runtime_settings_default_to_one_concurrent_parse() -> None: + """A standalone process should admit one expensive parse by default.""" + + settings = RuntimeSettings() + + assert settings.max_concurrent_parses == 1 From 1efc91810c3cb0658889194170510e8ecc54e338 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 10:53:21 +0900 Subject: [PATCH 20/75] feat(capacity): default each process to one parse --- src/newsdom_api/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/newsdom_api/config.py b/src/newsdom_api/config.py index 34bd05f9..5ad72867 100644 --- a/src/newsdom_api/config.py +++ b/src/newsdom_api/config.py @@ -38,6 +38,7 @@ class RuntimeSettings: authentication_mode: AuthenticationMode = AuthenticationMode.REQUIRED runtime_profile: RuntimeProfile = RuntimeProfile.PRODUCTION api_token: str | None = field(default=None, repr=False) + max_concurrent_parses: int = 1 def __post_init__(self) -> None: """Normalize secrets once and reject unsafe direct construction.""" From ac7a53889e3b04a08b6ca2588cf51a7e55aff912 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 10:55:14 +0900 Subject: [PATCH 21/75] test(capacity): require configured parser budget --- tests/test_parse_capacity.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/test_parse_capacity.py b/tests/test_parse_capacity.py index 58435b16..be353069 100644 --- a/tests/test_parse_capacity.py +++ b/tests/test_parse_capacity.py @@ -1,6 +1,6 @@ """Contracts for bounded parser admission control.""" -from newsdom_api.config import RuntimeSettings +from newsdom_api.config import RuntimeSettings, load_runtime_settings def test_runtime_settings_default_to_one_concurrent_parse() -> None: @@ -9,3 +9,11 @@ def test_runtime_settings_default_to_one_concurrent_parse() -> None: settings = RuntimeSettings() assert settings.max_concurrent_parses == 1 + + +def test_runtime_settings_load_configured_parse_capacity() -> None: + """Operators should be able to set a bounded per-process parse budget.""" + + settings = load_runtime_settings({"NEWSDOM_MAX_CONCURRENT_PARSES": "8"}) + + assert settings.max_concurrent_parses == 8 From 1bb99ea6a979f54d2726d44a37eef4cbc5d4ac7b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 10:58:25 +0900 Subject: [PATCH 22/75] feat(capacity): load configured parser budget --- src/newsdom_api/config.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/newsdom_api/config.py b/src/newsdom_api/config.py index 5ad72867..dff9f0a7 100644 --- a/src/newsdom_api/config.py +++ b/src/newsdom_api/config.py @@ -10,7 +10,9 @@ API_TOKEN_ENV_VAR = "NEWSDOM_API_TOKEN" AUTH_MODE_ENV_VAR = "NEWSDOM_AUTH_MODE" RUNTIME_PROFILE_ENV_VAR = "NEWSDOM_RUNTIME_PROFILE" +MAX_CONCURRENT_PARSES_ENV_VAR = "NEWSDOM_MAX_CONCURRENT_PARSES" MAX_BEARER_HEADER_BYTES = 4096 +DEFAULT_MAX_CONCURRENT_PARSES = 1 class RuntimeConfigurationError(ValueError): @@ -38,7 +40,7 @@ class RuntimeSettings: authentication_mode: AuthenticationMode = AuthenticationMode.REQUIRED runtime_profile: RuntimeProfile = RuntimeProfile.PRODUCTION api_token: str | None = field(default=None, repr=False) - max_concurrent_parses: int = 1 + max_concurrent_parses: int = DEFAULT_MAX_CONCURRENT_PARSES def __post_init__(self) -> None: """Normalize secrets once and reject unsafe direct construction.""" @@ -132,9 +134,16 @@ def load_runtime_settings( RuntimeProfile.PRODUCTION.value, ) ) + max_concurrent_parses = int( + values.get( + MAX_CONCURRENT_PARSES_ENV_VAR, + str(DEFAULT_MAX_CONCURRENT_PARSES), + ).strip() + ) return RuntimeSettings( authentication_mode=authentication_mode, runtime_profile=runtime_profile, api_token=get_api_token(values), + max_concurrent_parses=max_concurrent_parses, ) From 7a8d2a0596e1182ba05075905515d4e800d22bdd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 11:00:15 +0900 Subject: [PATCH 23/75] test(capacity): require bounded parser budget values --- tests/test_parse_capacity.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/tests/test_parse_capacity.py b/tests/test_parse_capacity.py index be353069..35d995d0 100644 --- a/tests/test_parse_capacity.py +++ b/tests/test_parse_capacity.py @@ -1,6 +1,12 @@ """Contracts for bounded parser admission control.""" -from newsdom_api.config import RuntimeSettings, load_runtime_settings +import pytest + +from newsdom_api.config import ( + RuntimeConfigurationError, + RuntimeSettings, + load_runtime_settings, +) def test_runtime_settings_default_to_one_concurrent_parse() -> None: @@ -17,3 +23,19 @@ def test_runtime_settings_load_configured_parse_capacity() -> None: settings = load_runtime_settings({"NEWSDOM_MAX_CONCURRENT_PARSES": "8"}) assert settings.max_concurrent_parses == 8 + + +@pytest.mark.parametrize("raw_value", ["", "0", "129", "not-an-integer"]) +def test_runtime_settings_reject_invalid_environment_capacity(raw_value: str) -> None: + """Environment input must be an integer in the supported process range.""" + + with pytest.raises(RuntimeConfigurationError, match="NEWSDOM_MAX_CONCURRENT_PARSES"): + load_runtime_settings({"NEWSDOM_MAX_CONCURRENT_PARSES": raw_value}) + + +@pytest.mark.parametrize("capacity", [0, 129, True]) +def test_runtime_settings_reject_invalid_direct_capacity(capacity: object) -> None: + """Direct construction must preserve the same immutable capacity boundary.""" + + with pytest.raises(RuntimeConfigurationError, match="NEWSDOM_MAX_CONCURRENT_PARSES"): + RuntimeSettings(max_concurrent_parses=capacity) # type: ignore[arg-type] From 809682c8451d9ccdadc90cf9b6eb89e7bfb3e2af Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 11:01:51 +0900 Subject: [PATCH 24/75] test(capacity): cover non-integer direct budgets --- tests/test_parse_capacity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_parse_capacity.py b/tests/test_parse_capacity.py index 35d995d0..f6d069e3 100644 --- a/tests/test_parse_capacity.py +++ b/tests/test_parse_capacity.py @@ -33,7 +33,7 @@ def test_runtime_settings_reject_invalid_environment_capacity(raw_value: str) -> load_runtime_settings({"NEWSDOM_MAX_CONCURRENT_PARSES": raw_value}) -@pytest.mark.parametrize("capacity", [0, 129, True]) +@pytest.mark.parametrize("capacity", [0, 129, True, "3"]) def test_runtime_settings_reject_invalid_direct_capacity(capacity: object) -> None: """Direct construction must preserve the same immutable capacity boundary.""" From 923c782bc9f5719515fcd4891ab3dd39d779d4d1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 11:02:54 +0900 Subject: [PATCH 25/75] feat(capacity): validate bounded parser budgets --- src/newsdom_api/config.py | 45 +++++++++++++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/src/newsdom_api/config.py b/src/newsdom_api/config.py index dff9f0a7..458b39ad 100644 --- a/src/newsdom_api/config.py +++ b/src/newsdom_api/config.py @@ -13,6 +13,7 @@ MAX_CONCURRENT_PARSES_ENV_VAR = "NEWSDOM_MAX_CONCURRENT_PARSES" MAX_BEARER_HEADER_BYTES = 4096 DEFAULT_MAX_CONCURRENT_PARSES = 1 +MAX_MAX_CONCURRENT_PARSES = 128 class RuntimeConfigurationError(ValueError): @@ -33,6 +34,20 @@ class RuntimeProfile(str, Enum): DEVELOPMENT = "development" +def _validated_parse_capacity(value: object) -> int: + """Return one supported non-boolean per-process parse capacity.""" + + if ( + isinstance(value, bool) + or not isinstance(value, int) + or not 1 <= value <= MAX_MAX_CONCURRENT_PARSES + ): + raise RuntimeConfigurationError( + "Invalid value for NEWSDOM_MAX_CONCURRENT_PARSES; use an integer from 1 to 128" + ) + return value + + @dataclass(frozen=True, slots=True) class RuntimeSettings: """Validated settings frozen for the lifetime of one FastAPI application.""" @@ -53,6 +68,12 @@ def __post_init__(self) -> None: "Authentication can be disabled only in the development runtime profile" ) + object.__setattr__( + self, + "max_concurrent_parses", + _validated_parse_capacity(self.max_concurrent_parses), + ) + if self.api_token is None: return @@ -101,6 +122,22 @@ def _parse_enum_value( return raw +def _parse_capacity_value(source: Mapping[str, str]) -> int: + """Parse the bounded parser budget without echoing malformed input.""" + + raw = source.get( + MAX_CONCURRENT_PARSES_ENV_VAR, + str(DEFAULT_MAX_CONCURRENT_PARSES), + ).strip() + try: + parsed = int(raw) + except ValueError as exc: + raise RuntimeConfigurationError( + "Invalid value for NEWSDOM_MAX_CONCURRENT_PARSES; use an integer from 1 to 128" + ) from exc + return _validated_parse_capacity(parsed) + + def get_api_token(source: Mapping[str, str] | None = None) -> str | None: """Return a normalized bootstrap token without logging or exposing it.""" @@ -134,16 +171,10 @@ def load_runtime_settings( RuntimeProfile.PRODUCTION.value, ) ) - max_concurrent_parses = int( - values.get( - MAX_CONCURRENT_PARSES_ENV_VAR, - str(DEFAULT_MAX_CONCURRENT_PARSES), - ).strip() - ) return RuntimeSettings( authentication_mode=authentication_mode, runtime_profile=runtime_profile, api_token=get_api_token(values), - max_concurrent_parses=max_concurrent_parses, + max_concurrent_parses=_parse_capacity_value(values), ) From cc0b938c620c00d12c8cc7ba942eb77cc6cb3788 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 13:18:39 +0900 Subject: [PATCH 26/75] test(capacity): require app-local parser admission limiter --- tests/test_parse_admission.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 tests/test_parse_admission.py diff --git a/tests/test_parse_admission.py b/tests/test_parse_admission.py new file mode 100644 index 00000000..1213a26a --- /dev/null +++ b/tests/test_parse_admission.py @@ -0,0 +1,25 @@ +"""Contracts for process-local parser admission control.""" + +from newsdom_api.config import AuthenticationMode, RuntimeProfile, RuntimeSettings +from newsdom_api.main import create_app + + +def _development_settings(*, capacity: int) -> RuntimeSettings: + """Return explicit development settings for one admission-control test.""" + + return RuntimeSettings( + authentication_mode=AuthenticationMode.DISABLED, + runtime_profile=RuntimeProfile.DEVELOPMENT, + max_concurrent_parses=capacity, + ) + + +def test_application_binds_configured_process_local_parse_capacity() -> None: + """Each application should own the immutable parse capacity it was given.""" + + application = create_app( + _development_settings(capacity=3), + runtime_readiness_probe=lambda: True, + ) + + assert application.state.parse_admission_limiter.capacity == 3 From ae0c0562dcecbda855398157d93be83795193d44 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 13:22:23 +0900 Subject: [PATCH 27/75] feat(capacity): add application-local concurrency state --- src/newsdom_api/admission.py | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 src/newsdom_api/admission.py diff --git a/src/newsdom_api/admission.py b/src/newsdom_api/admission.py new file mode 100644 index 00000000..2edc5489 --- /dev/null +++ b/src/newsdom_api/admission.py @@ -0,0 +1,10 @@ +"""Application-local concurrency state for parser work.""" + +from dataclasses import dataclass + + +@dataclass(frozen=True, slots=True) +class ParseAdmissionLimiter: + """Store the immutable parser concurrency budget for one application.""" + + capacity: int From 57dd8873bd75d88cad3d806a722a4839f238fc89 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 13:25:39 +0900 Subject: [PATCH 28/75] feat(capacity): bind app-local parser admission state --- src/newsdom_api/main.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/newsdom_api/main.py b/src/newsdom_api/main.py index f61aafc2..cedeabcd 100644 --- a/src/newsdom_api/main.py +++ b/src/newsdom_api/main.py @@ -24,6 +24,7 @@ from pypdf import PdfReader from pypdf.errors import PdfReadError +from .admission import ParseAdmissionLimiter from .config import ( AuthenticationMode, MAX_BEARER_HEADER_BYTES, @@ -324,6 +325,9 @@ def create_app( }, ) application.state.runtime_settings = application_settings + application.state.parse_admission_limiter = ParseAdmissionLimiter( + application_settings.max_concurrent_parses + ) application.state.runtime_readiness_probe = ( runtime_readiness_probe or mineru_runtime_available ) @@ -375,4 +379,4 @@ def create_app( return application -app = create_app() +app = create_app() \ No newline at end of file From f8af6c1056d4e414876a51d54289d8eb00371c77 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 13:28:20 +0900 Subject: [PATCH 29/75] test(capacity): require non-waiting admission leases --- tests/test_parse_admission.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_parse_admission.py b/tests/test_parse_admission.py index 1213a26a..58ac2f99 100644 --- a/tests/test_parse_admission.py +++ b/tests/test_parse_admission.py @@ -23,3 +23,21 @@ def test_application_binds_configured_process_local_parse_capacity() -> None: ) assert application.state.parse_admission_limiter.capacity == 3 + + +def test_limiter_rejects_excess_work_without_waiting_and_recovers() -> None: + """One released lease should immediately restore one unit of capacity.""" + + application = create_app( + _development_settings(capacity=1), + runtime_readiness_probe=lambda: True, + ) + limiter = application.state.parse_admission_limiter + + assert limiter.try_acquire() is True + assert limiter.try_acquire() is False + + limiter.release() + + assert limiter.try_acquire() is True + limiter.release() From 9cd6382ac721accc4e6e7253dbe01acd8d2db02c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 13:31:36 +0900 Subject: [PATCH 30/75] feat(capacity): implement non-waiting parser leases --- src/newsdom_api/admission.py | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/src/newsdom_api/admission.py b/src/newsdom_api/admission.py index 2edc5489..72fe47de 100644 --- a/src/newsdom_api/admission.py +++ b/src/newsdom_api/admission.py @@ -1,10 +1,31 @@ """Application-local concurrency state for parser work.""" -from dataclasses import dataclass +from threading import BoundedSemaphore -@dataclass(frozen=True, slots=True) class ParseAdmissionLimiter: - """Store the immutable parser concurrency budget for one application.""" + """Bound concurrent parser work without queueing excess requests.""" - capacity: int + __slots__ = ("_capacity", "_semaphore") + + def __init__(self, capacity: int) -> None: + """Create a limiter with an immutable number of parser leases.""" + + self._capacity = capacity + self._semaphore = BoundedSemaphore(capacity) + + @property + def capacity(self) -> int: + """Return the immutable number of parser leases.""" + + return self._capacity + + def try_acquire(self) -> bool: + """Acquire one parser lease immediately or report saturation.""" + + return self._semaphore.acquire(blocking=False) + + def release(self) -> None: + """Return one previously acquired parser lease.""" + + self._semaphore.release() From 5b500707c23e317c0ace7c0e49fa578b21a7caad Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 13:33:24 +0900 Subject: [PATCH 31/75] test(capacity): require pre-body saturation response --- tests/test_parse_admission.py | 65 ++++++++++++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) diff --git a/tests/test_parse_admission.py b/tests/test_parse_admission.py index 58ac2f99..e8d1ab03 100644 --- a/tests/test_parse_admission.py +++ b/tests/test_parse_admission.py @@ -1,7 +1,14 @@ """Contracts for process-local parser admission control.""" +import json +from typing import Awaitable, Callable + +import pytest +from fastapi import FastAPI, Request, Response +from fastapi.responses import JSONResponse + from newsdom_api.config import AuthenticationMode, RuntimeProfile, RuntimeSettings -from newsdom_api.main import create_app +from newsdom_api.main import create_app, security_boundary_middleware def _development_settings(*, capacity: int) -> RuntimeSettings: @@ -14,6 +21,32 @@ def _development_settings(*, capacity: int) -> RuntimeSettings: ) +def _parse_request(application: FastAPI) -> Request: + """Build a body-hostile parse request for direct middleware verification.""" + + async def receive() -> dict[str, object]: + raise AssertionError("saturated admission must not read the request body") + + return Request( + { + "type": "http", + "asgi": {"version": "3.0", "spec_version": "2.3"}, + "http_version": "1.1", + "method": "POST", + "scheme": "http", + "path": "/parse", + "raw_path": b"/parse", + "query_string": b"", + "headers": [], + "client": ("testclient", 50000), + "server": ("testserver", 80), + "root_path": "", + "app": application, + }, + receive, + ) + + def test_application_binds_configured_process_local_parse_capacity() -> None: """Each application should own the immutable parse capacity it was given.""" @@ -41,3 +74,33 @@ def test_limiter_rejects_excess_work_without_waiting_and_recovers() -> None: assert limiter.try_acquire() is True limiter.release() + + +@pytest.mark.asyncio +async def test_saturation_returns_fixed_429_before_downstream_or_body_work() -> None: + """Excess parser work should fail before multipart or parser execution.""" + + application = create_app( + _development_settings(capacity=1), + runtime_readiness_probe=lambda: True, + ) + limiter = application.state.parse_admission_limiter + assert limiter.try_acquire() is True + downstream_called = False + + async def downstream(_request: Request) -> Response: + nonlocal downstream_called + downstream_called = True + return JSONResponse({"status": "unexpected"}) + + response = await security_boundary_middleware( + _parse_request(application), + downstream, + ) + + assert response.status_code == 429 + assert json.loads(response.body) == {"detail": "Too Many Requests"} + assert response.headers["Retry-After"] == "1" + assert response.headers["Cache-Control"] == "no-store, no-cache, max-age=0" + assert downstream_called is False + limiter.release() From 925b7919e3f7790fdd34d4b6cb490942eda3e1c8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 13:37:42 +0900 Subject: [PATCH 32/75] feat(capacity): reject saturated parser work before body reads --- src/newsdom_api/main.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/newsdom_api/main.py b/src/newsdom_api/main.py index cedeabcd..db216b35 100644 --- a/src/newsdom_api/main.py +++ b/src/newsdom_api/main.py @@ -48,7 +48,9 @@ PAYLOAD_TOO_LARGE_DETAIL = "Payload Too Large" INVALID_PARSE_PARAMS_DETAIL = "Invalid parse parameters" UNAUTHORIZED_DETAIL = "Unauthorized" +TOO_MANY_REQUESTS_DETAIL = "Too Many Requests" SERVICE_UNAVAILABLE_DETAIL = "Service Unavailable" +RETRY_AFTER_SECONDS = "1" LOGGER = logging.getLogger("newsdom_api") BEARER_SCHEME = HTTPBearer(auto_error=False, scheme_name="BearerAuth") @@ -109,6 +111,16 @@ def _unauthorized_response() -> JSONResponse: ) +def _too_many_requests_response() -> JSONResponse: + """Return one fixed response for a saturated parser process.""" + + return JSONResponse( + status_code=429, + content={"detail": TOO_MANY_REQUESTS_DETAIL}, + headers={"Retry-After": RETRY_AFTER_SECONDS}, + ) + + def _parse_access_failure(request: Request) -> JSONResponse | None: """Validate `/parse` authorization before multipart upload parsing begins.""" @@ -141,12 +153,22 @@ async def security_boundary_middleware( request: Request, call_next: Callable, ) -> Response: - """Enforce parser authorization before reading the request body and add headers.""" + """Enforce parser access boundaries before reading the request body.""" if request.method == "POST" and request.scope.get("path") == "/parse": failure = _parse_access_failure(request) if failure is not None: return _apply_security_headers(failure, request) + + limiter: ParseAdmissionLimiter = request.app.state.parse_admission_limiter + if not limiter.try_acquire(): + return _apply_security_headers(_too_many_requests_response(), request) + try: + response = await call_next(request) + finally: + limiter.release() + return _apply_security_headers(response, request) + response = await call_next(request) return _apply_security_headers(response, request) @@ -371,6 +393,7 @@ def create_app( 413: {"description": PAYLOAD_TOO_LARGE_DETAIL}, 415: {"description": UNSUPPORTED_MEDIA_DETAIL}, 422: {"description": INVALID_PARSE_PARAMS_DETAIL}, + 429: {"description": TOO_MANY_REQUESTS_DETAIL}, 502: {"description": "Bad Gateway"}, 503: {"description": SERVICE_UNAVAILABLE_DETAIL}, }, From 056c88644ea906a387b5b97ecaa05790e852afda Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 13:38:27 +0900 Subject: [PATCH 33/75] test(capacity): keep saturation contract lint-clean --- tests/test_parse_admission.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_parse_admission.py b/tests/test_parse_admission.py index e8d1ab03..3a40d73c 100644 --- a/tests/test_parse_admission.py +++ b/tests/test_parse_admission.py @@ -1,7 +1,6 @@ """Contracts for process-local parser admission control.""" import json -from typing import Awaitable, Callable import pytest from fastapi import FastAPI, Request, Response From 0accc51788344e66282e3a3b44b8559a23063df5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 13:40:36 +0900 Subject: [PATCH 34/75] test(capacity): prove lease safety and burst backpressure --- tests/test_parse_admission.py | 214 +++++++++++++++++++++++++++++++++- 1 file changed, 211 insertions(+), 3 deletions(-) diff --git a/tests/test_parse_admission.py b/tests/test_parse_admission.py index 3a40d73c..af262a8f 100644 --- a/tests/test_parse_admission.py +++ b/tests/test_parse_admission.py @@ -1,5 +1,6 @@ """Contracts for process-local parser admission control.""" +import asyncio import json import pytest @@ -20,11 +21,26 @@ def _development_settings(*, capacity: int) -> RuntimeSettings: ) -def _parse_request(application: FastAPI) -> Request: +def _production_settings(*, capacity: int) -> RuntimeSettings: + """Return fail-closed production settings with one fixed parser token.""" + + return RuntimeSettings( + authentication_mode=AuthenticationMode.REQUIRED, + runtime_profile=RuntimeProfile.PRODUCTION, + api_token="admission-test-token", + max_concurrent_parses=capacity, + ) + + +def _parse_request( + application: FastAPI, + *, + headers: list[tuple[bytes, bytes]] | None = None, +) -> Request: """Build a body-hostile parse request for direct middleware verification.""" async def receive() -> dict[str, object]: - raise AssertionError("saturated admission must not read the request body") + raise AssertionError("admission middleware must not read the request body") return Request( { @@ -36,7 +52,7 @@ async def receive() -> dict[str, object]: "path": "/parse", "raw_path": b"/parse", "query_string": b"", - "headers": [], + "headers": headers or [], "client": ("testclient", 50000), "server": ("testserver", 80), "root_path": "", @@ -75,6 +91,26 @@ def test_limiter_rejects_excess_work_without_waiting_and_recovers() -> None: limiter.release() +def test_applications_do_not_share_parser_leases() -> None: + """One saturated application must not reduce another application's capacity.""" + + first = create_app( + _development_settings(capacity=1), + runtime_readiness_probe=lambda: True, + ) + second = create_app( + _development_settings(capacity=1), + runtime_readiness_probe=lambda: True, + ) + + assert first.state.parse_admission_limiter.try_acquire() is True + assert first.state.parse_admission_limiter.try_acquire() is False + assert second.state.parse_admission_limiter.try_acquire() is True + + first.state.parse_admission_limiter.release() + second.state.parse_admission_limiter.release() + + @pytest.mark.asyncio async def test_saturation_returns_fixed_429_before_downstream_or_body_work() -> None: """Excess parser work should fail before multipart or parser execution.""" @@ -103,3 +139,175 @@ async def downstream(_request: Request) -> Response: assert response.headers["Cache-Control"] == "no-store, no-cache, max-age=0" assert downstream_called is False limiter.release() + + +@pytest.mark.asyncio +async def test_successful_downstream_work_returns_its_parser_lease() -> None: + """A completed parser request should make its lease immediately reusable.""" + + application = create_app( + _development_settings(capacity=1), + runtime_readiness_probe=lambda: True, + ) + limiter = application.state.parse_admission_limiter + + async def downstream(_request: Request) -> Response: + return JSONResponse({"status": "complete"}) + + response = await security_boundary_middleware( + _parse_request(application), + downstream, + ) + + assert response.status_code == 200 + assert limiter.try_acquire() is True + limiter.release() + + +@pytest.mark.asyncio +async def test_exceptional_downstream_work_returns_its_parser_lease() -> None: + """A parser exception must not permanently consume process capacity.""" + + application = create_app( + _development_settings(capacity=1), + runtime_readiness_probe=lambda: True, + ) + limiter = application.state.parse_admission_limiter + + async def downstream(_request: Request) -> Response: + raise RuntimeError("parser failed") + + with pytest.raises(RuntimeError, match="parser failed"): + await security_boundary_middleware( + _parse_request(application), + downstream, + ) + + assert limiter.try_acquire() is True + limiter.release() + + +@pytest.mark.asyncio +async def test_cancelled_downstream_work_returns_its_parser_lease() -> None: + """Request cancellation must not leak a parser lease.""" + + application = create_app( + _development_settings(capacity=1), + runtime_readiness_probe=lambda: True, + ) + limiter = application.state.parse_admission_limiter + + async def downstream(_request: Request) -> Response: + raise asyncio.CancelledError + + with pytest.raises(asyncio.CancelledError): + await security_boundary_middleware( + _parse_request(application), + downstream, + ) + + assert limiter.try_acquire() is True + limiter.release() + + +@pytest.mark.asyncio +async def test_authentication_failure_does_not_consume_parser_capacity() -> None: + """Unauthorized requests should be rejected before admission accounting.""" + + application = create_app( + _production_settings(capacity=2), + runtime_readiness_probe=lambda: True, + ) + downstream_called = False + + async def downstream(_request: Request) -> Response: + nonlocal downstream_called + downstream_called = True + return JSONResponse({"status": "unexpected"}) + + response = await security_boundary_middleware( + _parse_request(application), + downstream, + ) + limiter = application.state.parse_admission_limiter + + assert response.status_code == 401 + assert downstream_called is False + assert limiter.try_acquire() is True + assert limiter.try_acquire() is True + assert limiter.try_acquire() is False + limiter.release() + limiter.release() + + +@pytest.mark.asyncio +async def test_thirty_two_requests_observe_a_four_request_capacity_bound() -> None: + """A realistic burst should admit four requests and reject the other 28.""" + + capacity = 4 + application = create_app( + _development_settings(capacity=capacity), + runtime_readiness_probe=lambda: True, + ) + all_admitted = asyncio.Event() + release_admitted = asyncio.Event() + admitted_count = 0 + + async def held_downstream(_request: Request) -> Response: + nonlocal admitted_count + admitted_count += 1 + if admitted_count == capacity: + all_admitted.set() + await release_admitted.wait() + return JSONResponse({"status": "complete"}) + + async def unexpected_downstream(_request: Request) -> Response: + raise AssertionError("saturated requests must not reach downstream parsing") + + admitted_tasks = [ + asyncio.create_task( + security_boundary_middleware( + _parse_request(application), + held_downstream, + ) + ) + for _ in range(capacity) + ] + await asyncio.wait_for(all_admitted.wait(), timeout=1.0) + + saturated_responses = await asyncio.gather( + *[ + security_boundary_middleware( + _parse_request(application), + unexpected_downstream, + ) + for _ in range(32 - capacity) + ] + ) + + assert [response.status_code for response in saturated_responses] == [429] * 28 + assert all(response.headers["Retry-After"] == "1" for response in saturated_responses) + + release_admitted.set() + admitted_responses = await asyncio.gather(*admitted_tasks) + + assert [response.status_code for response in admitted_responses] == [200] * capacity + limiter = application.state.parse_admission_limiter + reacquired = [limiter.try_acquire() for _ in range(capacity)] + assert reacquired == [True] * capacity + assert limiter.try_acquire() is False + for _ in range(capacity): + limiter.release() + + +def test_openapi_documents_parser_saturation_response() -> None: + """The parser operation should publish the fixed 429 response contract.""" + + application = create_app( + _development_settings(capacity=1), + runtime_readiness_probe=lambda: True, + ) + + response_contract = application.openapi()["paths"]["/parse"]["post"]["responses"] + + assert response_contract["429"]["description"] == "Too Many Requests" From 0255c537915d6f7625c84cfddc372a0f8ca51731 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 13:46:32 +0900 Subject: [PATCH 35/75] test(capacity): require operator and standards evidence --- tests/test_parse_capacity.py | 71 ++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/tests/test_parse_capacity.py b/tests/test_parse_capacity.py index f6d069e3..8a381912 100644 --- a/tests/test_parse_capacity.py +++ b/tests/test_parse_capacity.py @@ -1,6 +1,9 @@ """Contracts for bounded parser admission control.""" +from pathlib import Path + import pytest +import yaml from newsdom_api.config import ( RuntimeConfigurationError, @@ -8,6 +11,14 @@ load_runtime_settings, ) +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] + + +def _text(path: str) -> str: + """Read one repository-relative UTF-8 artifact.""" + + return (REPOSITORY_ROOT / path).read_text(encoding="utf-8") + def test_runtime_settings_default_to_one_concurrent_parse() -> None: """A standalone process should admit one expensive parse by default.""" @@ -39,3 +50,63 @@ def test_runtime_settings_reject_invalid_direct_capacity(capacity: object) -> No with pytest.raises(RuntimeConfigurationError, match="NEWSDOM_MAX_CONCURRENT_PARSES"): RuntimeSettings(max_concurrent_parses=capacity) # type: ignore[arg-type] + + +def test_deployment_examples_publish_the_per_process_capacity_budget() -> None: + """Every production example should expose one explicit parser capacity.""" + + assert "NEWSDOM_MAX_CONCURRENT_PARSES=1" in _text(".env.example") + + compose = yaml.safe_load(_text("docker-compose.yml")) + compose_environment = compose["services"]["newsdom-api"]["environment"] + assert compose_environment["NEWSDOM_MAX_CONCURRENT_PARSES"] == "1" + + documents = list( + yaml.safe_load_all(_text("docs/operations/kubernetes-deployment.yaml")) + ) + deployment = next(document for document in documents if document["kind"] == "Deployment") + container = deployment["spec"]["template"]["spec"]["containers"][0] + environment = {entry["name"]: entry for entry in container["env"]} + assert environment["NEWSDOM_MAX_CONCURRENT_PARSES"]["value"] == "1" + + +def test_operator_docs_explain_non_waiting_backpressure_and_replica_capacity() -> None: + """Operators need enough context to size capacity without hidden queues.""" + + combined = "\n".join( + ( + _text("README.md"), + _text("ARCHITECTURE.md"), + _text("docs/operations/deploy-runbook.md"), + _text("CHANGELOG.md"), + ) + ) + + for expected in ( + "NEWSDOM_MAX_CONCURRENT_PARSES", + "429 Too Many Requests", + "Retry-After: 1", + "per process", + "replica", + "before the multipart body", + ): + assert expected in combined + + +def test_capacity_doctoring_records_primary_sources_in_apa_seventh_style() -> None: + """The decision record should preserve standards and security rationale.""" + + doctoring = _text("docs/doctoring/bounded-parser-admission.md") + + for expected in ( + "Nottingham, M., & Fielding, R. (2012)", + "Fielding, R., Nottingham, M., & Reschke, J. (2022)", + "OWASP Foundation. (2023)", + "Python Software Foundation. (2026)", + "RFC 6585", + "RFC 9111", + "API4:2023", + "BoundedSemaphore", + "Rollback", + ): + assert expected in doctoring From 6f5d0eb045381e1769e89e32336580a9439acffb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 13:49:38 +0900 Subject: [PATCH 36/75] test(auth): isolate token concurrency from parser capacity --- tests/test_auth.py | 87 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 74 insertions(+), 13 deletions(-) diff --git a/tests/test_auth.py b/tests/test_auth.py index 3dc8311b..1bc71935 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -38,6 +38,7 @@ def _settings( token: str | None = "s3cret-token", mode: AuthenticationMode = AuthenticationMode.REQUIRED, profile: RuntimeProfile = RuntimeProfile.PRODUCTION, + max_concurrent_parses: int = 1, ) -> RuntimeSettings: """Build one explicit immutable settings object for an application test.""" @@ -45,6 +46,7 @@ def _settings( authentication_mode=mode, runtime_profile=profile, api_token=token, + max_concurrent_parses=max_concurrent_parses, ) @@ -346,7 +348,8 @@ def test_concurrent_requests_cannot_switch_authentication_state( """Concurrent callers should observe one frozen token and mode.""" application = create_app( - _settings(token="fixed"), runtime_readiness_probe=lambda: True + _settings(token="fixed", max_concurrent_parses=20), + runtime_readiness_probe=lambda: True, ) def request(token: str) -> int: @@ -388,22 +391,80 @@ def test_get_api_token_normalizes_bootstrap_transport() -> None: assert get_api_token({}) is None assert get_api_token({API_TOKEN_ENV_VAR: " "}) is None - assert get_api_token({API_TOKEN_ENV_VAR: " padded-token\n"}) == "padded-token" + assert get_api_token({API_TOKEN_ENV_VAR: " mounted-secret\n"}) == "mounted-secret" -def test_default_runtime_settings_are_required_production() -> None: - """Missing mode variables must select the fail-closed production contract.""" +def test_environment_settings_are_snapshotted_once( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Loading settings once should isolate them from later environment mutation.""" - settings = load_runtime_settings({API_TOKEN_ENV_VAR: "token"}) + environment = { + AUTH_MODE_ENV_VAR: "required", + RUNTIME_PROFILE_ENV_VAR: "production", + API_TOKEN_ENV_VAR: "initial-token", + } + settings = load_runtime_settings(environment) + environment[API_TOKEN_ENV_VAR] = "changed-token" + monkeypatch.setenv(API_TOKEN_ENV_VAR, "process-token") - assert settings.authentication_mode is AuthenticationMode.REQUIRED - assert settings.runtime_profile is RuntimeProfile.PRODUCTION - assert settings.authentication_ready is True + assert settings.api_token == "initial-token" -def test_config_module_exposes_versioned_environment_contract() -> None: - """Deployment tooling should use the exact documented environment names.""" +def test_default_factory_loads_settings_once_per_application( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The factory should evaluate runtime configuration only during app creation.""" + + calls = {"count": 0} + frozen = _settings() + + def fake_loader(): + calls["count"] += 1 + return frozen + + monkeypatch.setattr("newsdom_api.main.load_runtime_settings", fake_loader) + application = create_app(runtime_readiness_probe=lambda: True) + + assert calls["count"] == 1 + assert application.state.runtime_settings is frozen + + +def test_compatibility_get_api_token_reads_process_environment( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The compatibility helper should keep its explicit process lookup contract.""" - assert config.API_TOKEN_ENV_VAR == "NEWSDOM_API_TOKEN" - assert config.AUTH_MODE_ENV_VAR == "NEWSDOM_AUTH_MODE" - assert config.RUNTIME_PROFILE_ENV_VAR == "NEWSDOM_RUNTIME_PROFILE" + monkeypatch.setenv(API_TOKEN_ENV_VAR, "compat-token") + + assert get_api_token() == "compat-token" + + +def test_security_headers_are_applied_to_unhandled_server_errors() -> None: + """Unhandled errors should keep the same response hardening as normal traffic.""" + + async def failing_route() -> None: + raise RuntimeError("internal path /tmp/private-token") + + application = create_app(_settings(), runtime_readiness_probe=lambda: True) + application.add_api_route("/boom", failing_route, methods=["GET"]) + response = TestClient(application, raise_server_exceptions=False).get("/boom") + + assert response.status_code == 500 + assert response.json() == {"detail": "Internal Server Error"} + assert response.headers["X-Content-Type-Options"] == "nosniff" + assert "private-token" not in response.text + + +def test_main_module_logs_missing_required_token_without_crashing( + caplog: pytest.LogCaptureFixture, +) -> None: + """Application creation should expose configuration failure only to operators.""" + + caplog.set_level(logging.ERROR, logger="newsdom_api") + create_app(_settings(token=None), runtime_readiness_probe=lambda: True) + + assert "Parser authentication configuration is unavailable" in caplog.text + assert "token" not in caplog.text.lower().replace( + "parser authentication configuration is unavailable", "" + ) From fe1da17d0d6b5ee93133c3005e69d229a0226887 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 13:49:58 +0900 Subject: [PATCH 37/75] docs(capacity): publish the default process budget --- .env.example | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.env.example b/.env.example index e18ae5f8..fce7f0a4 100644 --- a/.env.example +++ b/.env.example @@ -3,5 +3,7 @@ NEWSDOM_AUTH_MODE=required NEWSDOM_RUNTIME_PROFILE=production # Generate a high-entropy token in the deployment secret store. Never commit it. NEWSDOM_API_TOKEN= +# Per-process non-waiting parser admission budget. Scale replicas separately. +NEWSDOM_MAX_CONCURRENT_PARSES=1 # Optional API-only image override for a bundled or mounted MinerU executable. # NEWSDOM_MINERU_BIN=/opt/mineru/bin/mineru From 28be99a53cb251cbf1398291c70a3c9019c11222 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 13:50:23 +0900 Subject: [PATCH 38/75] docs(capacity): configure compose parser admission --- docker-compose.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 19bdffa3..a290571f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,6 +15,9 @@ services: NEWSDOM_AUTH_MODE: required NEWSDOM_RUNTIME_PROFILE: production NEWSDOM_API_TOKEN: "${NEWSDOM_API_TOKEN:?set NEWSDOM_API_TOKEN}" + # One process-local MinerU lease is the conservative production default. + # Raise this only after measuring peak RSS/VRAM and parser latency. + NEWSDOM_MAX_CONCURRENT_PARSES: "1" # Point this at a bundled or mounted executable when the default API-only # image is used. The NVIDIA image already supplies its MinerU runtime. # NEWSDOM_MINERU_BIN: /opt/mineru/bin/mineru From cdcd9d7a326bbe87e9d3886fb59c9796e7024968 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 13:51:05 +0900 Subject: [PATCH 39/75] docs(capacity): configure Kubernetes parser leases --- docs/operations/kubernetes-deployment.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/operations/kubernetes-deployment.yaml b/docs/operations/kubernetes-deployment.yaml index f11f1f2c..79485ece 100644 --- a/docs/operations/kubernetes-deployment.yaml +++ b/docs/operations/kubernetes-deployment.yaml @@ -64,6 +64,10 @@ spec: secretKeyRef: name: newsdom-api-runtime key: api_token + - name: NEWSDOM_MAX_CONCURRENT_PARSES + # Capacity is per process and per replica; this two-replica + # example therefore admits at most two active parses at once. + value: "1" - name: NEWSDOM_MINERU_BIN value: /opt/mineru/bin/mineru volumeMounts: From 1d229c82e041fd430d10ff1be22bc454496e0caf Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 13:52:49 +0900 Subject: [PATCH 40/75] docs(capacity): explain parser overload behavior --- README.md | 47 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 79239a40..d1d15a28 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ submodule / sidecar in a larger system. available - Parsing `mode` defaults to `auto` so born-digital text PDFs skip forced OCR - Fail-closed bearer authentication on `/parse`; separate unauthenticated `/health` liveness and `/ready` traffic-readiness probes +- Bounded, non-waiting parser admission with a conservative one-parse default per process and an explicit `429 Too Many Requests` overload contract ## Quickstart @@ -42,7 +43,13 @@ On Windows, replace `.venv/bin/python` with `.venv\Scripts\python.exe`. ### Run +Configure the fail-closed production contract before starting the service: + ```bash +export NEWSDOM_AUTH_MODE=required +export NEWSDOM_RUNTIME_PROFILE=production +export NEWSDOM_API_TOKEN=$(openssl rand -hex 32) +export NEWSDOM_MAX_CONCURRENT_PARSES=1 uv run uvicorn --app-dir src newsdom_api.main:app --reload ``` @@ -50,7 +57,12 @@ uv run uvicorn --app-dir src newsdom_api.main:app --reload ```bash docker build -t newsdom-api . -docker run -p 8000:8000 newsdom-api +docker run -p 8000:8000 \ + -e NEWSDOM_AUTH_MODE=required \ + -e NEWSDOM_RUNTIME_PROFILE=production \ + -e NEWSDOM_API_TOKEN="$NEWSDOM_API_TOKEN" \ + -e NEWSDOM_MAX_CONCURRENT_PARSES=1 \ + newsdom-api ``` The default image exposes the REST API on port `8000` as a multi-arch service @@ -68,8 +80,8 @@ The default image ships the API service only and does not bundle the MinerU runt #### docker compose A production-oriented `docker-compose.yml` is provided for standalone/sidecar -use. It requires `NEWSDOM_API_TOKEN` before startup and its healthcheck targets -`/ready`: +use. It requires `NEWSDOM_API_TOKEN` before startup, sets the explicit +one-parse process budget, and its healthcheck targets `/ready`: ```bash docker compose up --build @@ -114,7 +126,9 @@ provide the CUDA user-space/runtime stack required by MinerU. ### Parse a PDF ```bash -curl -F "file=@sample.pdf" http://127.0.0.1:8000/parse +curl -F "file=@sample.pdf" \ + -H "Authorization: Bearer $NEWSDOM_API_TOKEN" \ + http://127.0.0.1:8000/parse ``` `/parse` accepts `multipart/form-data` with a required `file` part @@ -132,6 +146,7 @@ explicitly: ```bash curl -F "file=@sample.pdf" -F "language=japan" -F "mode=ocr" \ + -H "Authorization: Bearer $NEWSDOM_API_TOKEN" \ http://127.0.0.1:8000/parse ``` @@ -140,7 +155,7 @@ The accepted language contract follows the official MinerU canonicalizes `en`, `japan`, `chinese_cht`, and `latin` to `ch`; this sidecar performs the same normalization before launching the subprocess. -#### Authentication and readiness +#### Authentication, readiness, and parser capacity Parser authentication is required by default. This replaces the previous default-open behavior. Configure the explicit production contract and supply the @@ -150,6 +165,7 @@ secret from the deployment secret store: export NEWSDOM_AUTH_MODE=required export NEWSDOM_RUNTIME_PROFILE=production export NEWSDOM_API_TOKEN=$(openssl rand -hex 32) +export NEWSDOM_MAX_CONCURRENT_PARSES=1 uv run uvicorn --app-dir src newsdom_api.main:app curl -F "file=@sample.pdf" -H "Authorization: Bearer $NEWSDOM_API_TOKEN" \ http://127.0.0.1:8000/parse @@ -160,6 +176,21 @@ server token returns a fixed `503` before the upload body is processed. `GET /health` remains unauthenticated liveness. `GET /ready` succeeds only when the authentication configuration and MinerU runtime can accept traffic. +`NEWSDOM_MAX_CONCURRENT_PARSES` is an immutable integer from `1` through `128` +and defaults to `1`. It controls active MinerU work **per process**. Authentication +runs first. An authenticated request then obtains one non-waiting lease before the +multipart body is parsed or copied. When every lease is in use, the service returns +`429 Too Many Requests` with `Retry-After: 1` and `Cache-Control: no-store` before +the multipart body, temporary-file allocation, PDF validation, or MinerU execution. +The service deliberately does not create an unbounded in-process queue. + +Total nominal capacity is the process budget multiplied by the number of serving +processes and each replica. Start with one lease, measure peak RSS/VRAM and parser +latency on representative PDFs, and scale replicas before increasing per-process +capacity. Clients should apply bounded, jittered retries and honor `Retry-After: 1`. +The limit is availability protection rather than a substitute for gateway-level +per-tenant rate limits or cluster resource quotas. + An isolated local development-only bypass is available only with the explicit pair below: @@ -174,8 +205,8 @@ working secret or the previous release instead. Each request is written to a request-scoped temporary directory before MinerU runs, and those temporary files are removed after the response completes. -Sanitized parse failures return `503 MinerU runtime unavailable` when the -runtime cannot be executed and `502 MinerU output was incomplete` when MinerU +Sanitized parse failures return `503 Service Unavailable` when the +runtime cannot be executed and `502 Bad Gateway` when MinerU finishes without the required output artifacts. ### Run tests @@ -228,6 +259,6 @@ Repository branch workflow is documented in `docs/workflow/git-flow.md`. ## Repository layout -- `src/newsdom_api/`: API, MinerU wrapper, DOM builder, synthetic fixture generator +- `src/newsdom_api/`: API, MinerU wrapper, admission control, DOM builder, synthetic fixture generator - `tests/`: unit tests and committed synthetic fixtures - `tools/`: local maintenance utilities From a0a2e22f068e299d72adb12f806926967be2f7d2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 13:53:34 +0900 Subject: [PATCH 41/75] docs(capacity): record the admission architecture --- ARCHITECTURE.md | 131 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 95 insertions(+), 36 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 9a87b80d..8f1a7f72 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -2,54 +2,113 @@ ## Runtime shape -`newsdom-api` is a small service-oriented Python application with a -thin FastAPI entrypoint and explicit separation between request -orchestration, MinerU process execution, and DOM normalization. +`newsdom-api` is a small service-oriented Python application with a thin +FastAPI entrypoint and explicit separation between request security, +process-local admission, MinerU process execution, and DOM normalization. The +same application can run as a standalone service, a sidecar, or a bounded MSA +module behind an API gateway. ## Primary modules -- `src/newsdom_api/main.py` exposes `/health` and `/parse` - through FastAPI. -- `src/newsdom_api/service.py` orchestrates PDF parsing, - temporary files, and response construction. -- `src/newsdom_api/mineru_runner.py` shells out to the MinerU CLI, - collects JSON outputs, and translates runtime or incomplete-output - failures into typed sanitized exceptions. -- `src/newsdom_api/dom_builder.py` converts MinerU `content_list` - blocks plus page model metadata into the canonical NewsDOM response - model. +- `src/newsdom_api/main.py` exposes `/health`, `/ready`, and `/parse` through + FastAPI. Its outer middleware authenticates and admits parser work before the + multipart body is read. +- `src/newsdom_api/config.py` snapshots immutable runtime authentication, + profile, executable, and per-process capacity settings during application + creation. +- `src/newsdom_api/admission.py` owns the process-local non-waiting parser lease + pool. `ParseAdmissionLimiter` uses `threading.BoundedSemaphore` so excess + release is treated as a programming error rather than silently inflating + capacity. +- `src/newsdom_api/service.py` orchestrates PDF parsing and response + construction. +- `src/newsdom_api/mineru_runner.py` shells out to the MinerU CLI, collects JSON + outputs, and translates runtime or incomplete-output failures into typed + sanitized exceptions. +- `src/newsdom_api/dom_builder.py` converts MinerU `content_list` blocks plus + page model metadata into the canonical NewsDOM response model. - `src/newsdom_api/schemas.py` defines the public response schema. -- `src/newsdom_api/synthetic.py` and - `src/newsdom_api/equivalence.py` support synthetic fixture - generation and structural comparisons. +- `src/newsdom_api/synthetic.py` and `src/newsdom_api/equivalence.py` support + synthetic fixture generation and structural comparisons. ## Request flow -1. `src/newsdom_api/main.py` receives an uploaded PDF. -2. `src/newsdom_api/service.py` writes the upload to a temporary - workspace and calls MinerU. -3. `src/newsdom_api/mineru_runner.py` resolves the executable, runs - the OCR pipeline, loads generated JSON artifacts, and raises typed - sanitized errors for runtime-unavailable or incomplete-output cases. -4. `src/newsdom_api/dom_builder.py` normalizes OCR blocks into the - canonical response while preserving page-aware structure from - MinerU model metadata. -5. FastAPI returns typed JSON from `src/newsdom_api/schemas.py` and - maps MinerU runtime failures to 503 and incomplete output to 502. +```mermaid +flowchart LR + C[Client] --> A[Raw bearer authentication] + A -->|invalid| U[401 or configuration 503] + A -->|valid| L[Process-local non-waiting lease] + L -->|saturated| R[429 Too Many Requests\nRetry-After: 1] + L -->|admitted| M[Multipart body and upload limits] + M --> P[PDF structural validation] + P --> T[Temporary file] + T --> X[MinerU execution] + X --> D[Canonical DOM normalization] + D --> J[Typed JSON response] + J --> F[Lease release] + X -->|error or cancellation| F +``` + +1. `src/newsdom_api/main.py` validates the raw Authorization header without + decoding duplicate or non-ASCII credentials into a lossy string. +2. An authenticated `POST /parse` attempts one lease from + `ParseAdmissionLimiter` **before the multipart body** is parsed. The attempt + never waits. Saturation returns `429 Too Many Requests`, `Retry-After: 1`, + and non-cacheable response headers. +3. The route validates MIME type, form values, declared and streamed size, and + PDF structure before parser execution. +4. `src/newsdom_api/service.py` and `src/newsdom_api/mineru_runner.py` execute + MinerU under the configured runtime and error boundaries. +5. `src/newsdom_api/dom_builder.py` normalizes OCR blocks into the canonical + response while preserving page-aware structure from MinerU model metadata. +6. The outer `finally` boundary releases the parser lease after success, + validation failure, backend failure, request cancellation, or another + exception. +7. FastAPI returns typed JSON from `src/newsdom_api/schemas.py` and maps MinerU + runtime failures to 503 and incomplete output to 502. + +## Capacity model + +`NEWSDOM_MAX_CONCURRENT_PARSES` is immutable after application creation, accepts +`1..128`, and defaults to `1`. The budget is **per process**, not global. A +single-process replica configured with `N` can execute at most `N` MinerU jobs +at once. A deployment with `R` replicas and one serving process in each has a +nominal upper bound of `R ร— N`; additional gateway, scheduler, CPU, memory, +VRAM, and storage constraints can reduce effective throughput. + +The application intentionally does not wait for a parser slot. An internal +queue would already retain request bodies and open connections, hide overload +from the orchestrator, and make memory consumption proportional to demand. +The fixed 429 contract instead keeps queue ownership at a gateway or durable +job service that can implement tenant policy, persistence, cancellation, and +bounded retry semantics. + +## Liveness and readiness + +`GET /health` proves only that the web process is live. `GET /ready` proves that +fail-closed authentication configuration and the MinerU executable are +available. Parser saturation does not make a process unready: readiness is a +routing eligibility signal, whereas the admission response is instantaneous +load feedback for each request. A future durable asynchronous job API may add +queue-depth readiness policy without changing this synchronous endpoint's +non-waiting contract. ## Supporting systems -- `tests/fixtures` holds synthetic PDFs, JSON baselines, and - provenance notes; private reference inputs stay out of git. +- `tests/fixtures` holds synthetic PDFs, JSON baselines, and provenance notes; + private reference inputs stay out of git. - `manual/` is the published user manual rendered by MkDocs. -- `.github/workflows/` encodes CI, security scanning, Pages, - release, and image-delivery policy. -- `scripts/release/` builds release manifests and exports GitHub attestation bundles. +- `.github/workflows/` encodes CI, security scanning, Pages, release, and + image-delivery policy. +- `scripts/release/` builds release manifests and exports GitHub attestation + bundles. +- `docs/doctoring/bounded-parser-admission.md` records the resource-exhaustion, + HTTP, caching, implementation, rollback, and APA 7 evidence behind the + admission boundary. ## Delivery boundaries -- `develop` is the integration line for normal feature, fix, - and chore work. +- `develop` is the integration line for normal feature, fix, and chore work. - `main` is the stable release line that receives tagged releases. -- The service is production-grade only when code, docs, workflows, - and release evidence agree. +- The service is production-grade only when code, docs, workflows, and release + evidence agree. From 871f1322c69516c47f240078761abe4df5ed7ff8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 13:54:36 +0900 Subject: [PATCH 42/75] docs(capacity): add overload sizing and rollback runbook --- docs/operations/deploy-runbook.md | 72 ++++++++++++++++++++++++++++--- 1 file changed, 67 insertions(+), 5 deletions(-) diff --git a/docs/operations/deploy-runbook.md b/docs/operations/deploy-runbook.md index 760d0987..b01de439 100644 --- a/docs/operations/deploy-runbook.md +++ b/docs/operations/deploy-runbook.md @@ -16,6 +16,7 @@ release acceptance record. NEWSDOM_AUTH_MODE=required NEWSDOM_RUNTIME_PROFILE=production NEWSDOM_API_TOKEN= + NEWSDOM_MAX_CONCURRENT_PARSES=1 ``` 3. `uv run uvicorn --app-dir src newsdom_api.main:app --host 127.0.0.1 --port 8000` @@ -31,13 +32,14 @@ configuration and the MinerU executable are ready to accept traffic. ## Container smoke 1. `docker build -t newsdom-api:unreleased .` -2. Run with an explicit secret and production settings: +2. Run with an explicit secret, production settings, and process budget: ```bash docker run --rm -p 18080:8000 \ -e NEWSDOM_AUTH_MODE=required \ -e NEWSDOM_RUNTIME_PROFILE=production \ -e NEWSDOM_API_TOKEN="$NEWSDOM_API_TOKEN" \ + -e NEWSDOM_MAX_CONCURRENT_PARSES=1 \ newsdom-api:unreleased ``` @@ -53,6 +55,46 @@ runtime, so container smoke should treat `/parse` as an external-runtime check rather than a default-image contract. A container may be live while remaining unready; never route production traffic from `/health` alone. +## Parser admission and overload smoke + +`NEWSDOM_MAX_CONCURRENT_PARSES` accepts an integer from `1` through `128` and +defaults to `1`. The budget is **per process**. It is snapshotted when the +application is created and cannot be changed by mutating the environment later. +A deployment with `R` one-process replicas and a value of `N` has a nominal +active-parser ceiling of `R ร— N`. + +Authentication is evaluated before capacity. After successful authentication, +the service attempts one non-waiting parser lease **before the multipart body**, +temporary-file allocation, PDF validation, or MinerU. When all leases are in +use, the request receives: + +```http +HTTP/1.1 429 Too Many Requests +Retry-After: 1 +Cache-Control: no-store, no-cache, max-age=0 +``` + +No in-process waiting queue is created. The lease is released after success, +validation failure, MinerU failure, request cancellation, or another exception. +A gateway may add tenant-aware request-rate policy, but it must not convert the +synchronous endpoint into an unbounded buffered queue. + +To exercise the contract before rollout: + +1. Use a synthetic or non-sensitive representative PDF and configure a test + instance with `NEWSDOM_MAX_CONCURRENT_PARSES=1`. +2. Hold one authenticated parse active at the MinerU boundary. +3. Send a second authenticated parse and verify the fixed 429, exact + `Retry-After: 1`, and no-store cache policy. +4. Confirm no second temporary upload or MinerU process appears. +5. Release the first request, then verify the next parse is admitted. This proves + that success and failure paths do not leak leases. + +Start at one lease. Increase only after measuring peak RSS, peak VRAM when +applicable, temporary storage, p95/p99 parser latency, timeout rate, and output +accuracy on representative 1/5/20 MiB PDFs. Prefer additional replicas when one +parser already consumes a substantial share of the pod memory or GPU budget. + ## Kubernetes smoke 1. Replace `ghcr.io/contextualwisdomlab/newsdom-api:unreleased` in @@ -64,13 +106,17 @@ unready; never route production traffic from `/health` alone. 3. Ensure the MinerU executable and models are present in the deployed image or supplied through a separately reviewed read-only volume. The example exposes bounded writable `emptyDir` volumes only for `/tmp` and the runtime cache. -4. Apply the manifest and verify that the namespace enforces the Restricted Pod +4. Confirm `NEWSDOM_MAX_CONCURRENT_PARSES` is explicitly set per process. The + reference has two replicas at one lease each, for two nominal active parses. +5. Apply the manifest and verify that the namespace enforces the Restricted Pod Security profile. -5. Confirm the pod runs as UID/GID 10001 with `RuntimeDefault` seccomp, +6. Confirm the pod runs as UID/GID 10001 with `RuntimeDefault` seccomp, privilege escalation disabled, all Linux capabilities dropped, and a read-only root filesystem. -6. Verify `GET /health` for liveness and `GET /ready` for Service endpoint +7. Verify `GET /health` for liveness and `GET /ready` for Service endpoint readiness before sending authenticated parser traffic. +8. Run the overload smoke through the Service and verify excess work is spread + across replicas or receives the fixed 429 rather than accumulating in pods. The repository-owned GHCR path is the intended image source. Production cluster admission policy must allow only the reviewed organization/repository path and @@ -86,6 +132,8 @@ a cluster-specific registry trust policy. - After a tag push or manual dispatch, verify the GitHub Release contains `SHA256SUMS.txt`, `release-manifest.json`, and `*.intoto.jsonl` assets. - Record the accepted image digest and use that immutable digest in Kubernetes. +- Record the accepted parser capacity, replica count, load profile, peak resource + evidence, and rollback threshold in the deployment acceptance record. ## Failure handling @@ -94,19 +142,33 @@ a cluster-specific registry trust policy. - Expect `/parse` failures to stay sanitized: use generic client-facing details for `503 Service Unavailable` when the backend runtime cannot execute and `502 Bad Gateway` when required OCR artifacts are missing or invalid. +- Treat unexpected 429 growth as capacity pressure, not an authentication or + readiness failure. Inspect traffic, input-size mix, parser latency, RSS/VRAM, + replica availability, and downstream storage before changing the limit. - Reconcile the failure against `README.md`, `CHANGELOG.md`, package/OpenAPI versions, and the relevant workflow before closing the task. +## Rollback + +A safe admission rollback changes configuration or restores the previous release; +it never removes authentication, upload limits, or the non-waiting boundary. +Reduce `NEWSDOM_MAX_CONCURRENT_PARSES` to the last accepted value and restart the +processes so immutable settings are reloaded. If the new release itself is at +fault, restore the prior digest and its recorded capacity. Verify health, +readiness, one successful authenticated parse, one saturated 429, and subsequent +lease recovery before reopening traffic. + ## Fail-closed authentication migration for planned 0.3.0 Parser authentication is required by default. The previous default-open behavior -must not be carried into a production or shared deployment. Configure all three +must not be carried into a production or shared deployment. Configure all four settings before routing traffic: ```text NEWSDOM_AUTH_MODE=required NEWSDOM_RUNTIME_PROFILE=production NEWSDOM_API_TOKEN= +NEWSDOM_MAX_CONCURRENT_PARSES=1 ``` Use `GET /health` for process liveness and `GET /ready` for traffic readiness. From e5b81687a9a1a835426f74ad075d23ad54f453ee Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 13:57:14 +0900 Subject: [PATCH 43/75] docs(capacity): add bounded admission decision record --- docs/doctoring/bounded-parser-admission.md | 226 +++++++++++++++++++++ 1 file changed, 226 insertions(+) create mode 100644 docs/doctoring/bounded-parser-admission.md diff --git a/docs/doctoring/bounded-parser-admission.md b/docs/doctoring/bounded-parser-admission.md new file mode 100644 index 00000000..55a1e759 --- /dev/null +++ b/docs/doctoring/bounded-parser-admission.md @@ -0,0 +1,226 @@ +# Bounded parser admission + +## Decision + +NewsDOM applies a process-local, non-waiting concurrency boundary to the +expensive synchronous `/parse` path. The immutable setting +`NEWSDOM_MAX_CONCURRENT_PARSES` accepts an integer from `1` through `128` and +defaults to `1`. + +An authenticated request must acquire one parser lease before FastAPI or +Starlette parses the multipart body, before request bytes are copied to a +temporary file, before PDF structure validation, and before MinerU is invoked. +If no lease is available, the service immediately returns: + +```http +HTTP/1.1 429 Too Many Requests +Retry-After: 1 +Cache-Control: no-store, no-cache, max-age=0 +``` + +The service does not wait for a lease and does not create an in-process upload +queue. + +## Buyer-visible problem + +MinerU parsing can consume substantial CPU, memory, temporary storage, and GPU +resources. Without an admission boundary, a short request burst can begin many +multipart reads and parser processes at once. The resulting resource +competition can increase latency, trigger out-of-memory termination, exhaust +writable volumes, and make every request fail rather than rejecting only excess +work. + +OWASP API4:2023 identifies unrestricted consumption of CPU, memory, process, +payload, and third-party resources as an API availability and cost risk. The +control implemented here limits active parser work rather than relying on +client goodwill or a best-effort timeout after resources have already been +allocated. + +## Boundary ordering + +The access path is deliberately ordered as follows: + +```mermaid +flowchart LR + A[POST /parse] --> B[Raw bearer authentication] + B -->|invalid| C[401 or configuration 503] + B -->|valid| D[Non-waiting parser lease] + D -->|saturated| E[429 + Retry-After: 1 + no-store] + D -->|admitted| F[Multipart body] + F --> G[Upload and PDF validation] + G --> H[MinerU] + H --> I[DOM response] + I --> J[Lease release] + H -->|failure or cancellation| J +``` + +Authentication precedes admission so unauthenticated traffic cannot consume a +parser lease or infer capacity from overload responses. Admission precedes +multipart parsing so rejected work does not retain request bodies, allocate +request-scoped temporary files, validate PDFs, or enter MinerU. + +## HTTP semantics + +RFC 6585 defines `429 Too Many Requests` for request-rate or overload controls +and permits a `Retry-After` response field. NewsDOM uses an exact one-second +hint because the synchronous API has no durable queue position or completion +estimate. Clients should still use bounded jitter and stop retrying according +to their own deadline. + +RFC 6585 also states that 429 responses must not be stored by a cache. RFC 9111 +provides the `no-store` response directive. NewsDOM applies `Cache-Control: +no-store, no-cache, max-age=0` through the same security-header middleware used +for every parser response, preventing an intermediary from replaying a stale +overload decision after capacity has recovered. + +The response body is fixed and does not reveal the configured capacity, current +lease count, parser identity, queue depth, file metadata, internal paths, or +another tenant's activity. + +## Implementation + +`src/newsdom_api/admission.py` contains `ParseAdmissionLimiter`. Each FastAPI +application owns one limiter. The implementation uses +`threading.BoundedSemaphore` because the protected work executes through a +thread-backed synchronous parser boundary and requests may arrive from multiple +threads or event-loop tasks. + +`BoundedSemaphore` also treats release beyond the configured initial value as a +programming error. This is preferable to silently increasing capacity after a +double release. + +The middleware follows this invariant: + +```text +authenticate +โ†’ try_acquire without waiting +โ†’ call downstream application +โ†’ release in finally +``` + +The `finally` boundary returns the lease after a successful response, form or +PDF validation failure, MinerU runtime failure, incomplete MinerU output, +unhandled exception, or request cancellation. Authentication and service +configuration failures occur before acquisition and therefore do not require a +release. + +## Scope and MSA behavior + +The capacity is **per process**. It is not an organization-wide, cluster-wide, +repository-wide, or tenant-wide counter. If a replica has `P` serving processes +and each process is configured with `N`, that replica can nominally run at most +`P ร— N` parser jobs. With `R` identical replicas, the nominal deployment ceiling +is `R ร— P ร— N`. + +Actual safe capacity can be lower because pods share CPU, memory, storage, +model caches, and sometimes a GPU. A gateway may add identity-aware request +rates, quotas, fairness, and circuit breaking. A future durable job service may +own persistent queueing and cancellation. Neither layer should remove this +last-resort process boundary. + +The module remains usable independently in the standalone NewsDOM service and +when NewsDOM is imported as a sidecar or MSA component. It does not require a +shared database, distributed lock, Redis, or naruon runtime. + +## Configuration and deployment + +The conservative production default is: + +```text +NEWSDOM_MAX_CONCURRENT_PARSES=1 +``` + +The setting is parsed once during application creation and remains immutable. +Changing the environment of a running process does not resize the active +limiter. Operators must restart the serving process after an approved change. + +The repository-owned Docker Compose and Kubernetes examples explicitly set the +value to `1`. The Kubernetes example has two replicas, so it advertises two +nominal active parses while retaining independent failure domains. + +Raise the value only after measuring representative PDFs across the expected +size, page-count, language, OCR mode, and document-complexity distribution. +Record at least: + +- peak resident memory and, when applicable, peak VRAM; +- temporary-volume peak usage; +- p50, p95, and p99 parser latency; +- timeout, 429, 502, 503, and process-crash rates; +- output accuracy and structural-equivalence metrics; +- behavior when one replica is terminating or unavailable. + +Prefer additional replicas when one MinerU process already consumes a large +share of a pod's memory or GPU budget. + +## Verification evidence + +The regression suite proves: + +- immutable direct and environment configuration with a default of one; +- rejection of blank, non-integer, boolean, zero, and values above 128; +- application-local rather than global lease state; +- immediate saturation without a body read or downstream call; +- fixed 429 body, exact `Retry-After: 1`, and no-store response headers; +- lease recovery after success, exception, and cancellation; +- authentication failure without lease consumption; +- a realistic burst of 32 requests with capacity four, admitting four and + rejecting the remaining 28; +- OpenAPI publication of the 429 response; +- production deployment examples and operator documentation; +- 100% production statement and branch coverage and public docstrings. + +A live deployment acceptance test should additionally hold all configured +leases at the MinerU boundary, send one excess authenticated request, verify no +new temporary file or parser process is created, release one request, and prove +the next request is admitted. + +## Failure handling + +Unexpected 429 growth means the service is protecting itself. Operators should +inspect traffic composition, parser duration, replica health, CPU, RSS, VRAM, +temporary storage, and downstream dependencies before raising the limit. +Readiness should remain a dependency and security signal; ordinary short-lived +saturation does not make the process permanently unready. + +A leaked lease would reduce capacity until process restart. The test suite +covers every current response and cancellation path, while +`BoundedSemaphore` detects over-release. Monitoring should alert on sustained +429 rate, falling successful throughput, parser duration, and replica restart +rate rather than exposing internal lease counts to unauthenticated callers. + +## Rollback + +Rollback must preserve authentication, upload limits, and non-waiting +admission. Reduce `NEWSDOM_MAX_CONCURRENT_PARSES` to the last accepted value and +restart processes so the immutable setting is reloaded. If a release regression +is suspected, restore the previous verified image digest together with its +recorded capacity. + +Before reopening traffic, verify `/health`, `/ready`, one successful +authenticated parse, one intentional saturated 429, and successful admission +after the held request releases its lease. Do not roll back to an unbounded or +default-open parser. + +## Limitations and next slices + +This boundary does not provide durable asynchronous processing, tenant +fairness, distributed capacity, priority scheduling, or a job status and +cancellation API. Those capabilities belong in a separately bounded job +service. The synchronous endpoint intentionally exposes overload instead of +pretending to queue work it cannot persist or recover. + +## References + +Fielding, R., Nottingham, M., & Reschke, J. (2022). *HTTP caching* (RFC +9111). Internet Engineering Task Force. https://doi.org/10.17487/RFC9111 + +Nottingham, M., & Fielding, R. (2012). *Additional HTTP status codes* (RFC +6585). Internet Engineering Task Force. https://doi.org/10.17487/RFC6585 + +OWASP Foundation. (2023). *API4:2023 unrestricted resource consumption*. +*OWASP API Security Top 10*. +https://owasp.org/API-Security/editions/2023/en/0xa4-unrestricted-resource-consumption/ + +Python Software Foundation. (2026). *threadingโ€”Thread-based parallelism +(Python 3.10.20 documentation)*. +https://docs.python.org/3.10/library/threading.html From 02a1614b5c537a52fcabdf9448915476b53e31e6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 13:59:00 +0900 Subject: [PATCH 44/75] docs(capacity): record bounded parser admission --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f92c93b..bede6388 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - OpenAPI ์ œ๋ชฉ/์„ค๋ช…, README, `ArticleNode.headline` ๋ฌธ์„œ๋ฅผ ์ผ๋ฐ˜ ๋ฌธ์„œ์šฉ (section heading) ํ‘œํ˜„์œผ๋กœ ์žฌ๊ตฌ์„ฑํ•˜์—ฌ ํŠน์ • ์–ธ์–ด/์‹ ๋ฌธ ๊ฐ€์ •์„ ์†Œ๋น„์ž์—๊ฒŒ ๋…ธ์ถœํ•˜์ง€ ์•Š๋„๋ก ํ•จ. ์‘๋‹ต ์Šคํ‚ค๋งˆ ํ•„๋“œ๋Š” ํ•˜์œ„ ํ˜ธํ™˜์„ ์œ„ํ•ด ๋ณ€๊ฒฝํ•˜์ง€ ์•Š์Œ. ### Added +- Added immutable `NEWSDOM_MAX_CONCURRENT_PARSES` process admission with a conservative default of one. Authentication remains first; an authenticated request must acquire a non-waiting lease before the multipart body, temporary-file allocation, PDF validation, or MinerU. Saturation returns fixed `429 Too Many Requests`, `Retry-After: 1`, and no-store cache headers, while every success, error, exception, and cancellation path releases its lease. Docker Compose and Kubernetes examples publish the explicit per-process value and explain replica capacity. - [CLI] ๋‹จ์ผ NewsDOM JSON ํŒŒ์ผ์„ ํŽ˜์ด์ง€ ๋‹จ์œ„๋กœ ๋ถ„๋ฆฌํ•˜๋Š” `tools/split_dom.py` ๋„๊ตฌ๋ฅผ ์ถ”๊ฐ€ํ–ˆ์Šต๋‹ˆ๋‹ค. - [CLI] NewsDOM JSON ํŒŒ์ผ์˜ ๋ชจ๋“  ํ…์ŠคํŠธ ๋‚ด์šฉ์„ ๋งˆ์Šคํ‚นํ•˜์—ฌ ์ต๋ช…ํ™”ํ•˜๋Š” `tools/anonymize_dom.py` ๋„๊ตฌ๋ฅผ ์ถ”๊ฐ€ํ–ˆ์Šต๋‹ˆ๋‹ค. - `/parse`์— ๊ธฐ๋ณธ ํ•„์ˆ˜ bearer ์ธ์ฆ ๊ฒฝ๊ณ„๋ฅผ ์ถ”๊ฐ€ํ–ˆ์Šต๋‹ˆ๋‹ค. `NEWSDOM_AUTH_MODE=required`, `NEWSDOM_RUNTIME_PROFILE=production`, `NEWSDOM_API_TOKEN`์„ ๋ช…์‹œํ•ด์•ผ ํ•˜๋ฉฐ, ์ธ์ฆ ๋น„ํ™œ์„ฑํ™”๋Š” ๊ฒฉ๋ฆฌ๋œ development ํ”„๋กœํ•„์—์„œ๋งŒ ํ—ˆ์šฉ๋ฉ๋‹ˆ๋‹ค. `/health`๋Š” liveness ์ „์šฉ์œผ๋กœ ๋ฏธ์ธ์ฆ ์ƒํƒœ๋ฅผ ์œ ์ง€ํ•˜๊ณ  `/ready`๊ฐ€ ์ธ์ฆ ์„ค์ •๊ณผ MinerU ๊ฐ€์šฉ์„ฑ์„ ํ•จ๊ป˜ ๊ฒ€์ฆํ•ฉ๋‹ˆ๋‹ค. @@ -27,6 +28,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [CLI] ํŒŒ์‹ฑ๋œ NewsDOM JSON์—์„œ ์ˆœ์ˆ˜ ํ…์ŠคํŠธ ๋ฐ์ดํ„ฐ๋ฅผ ์ถ”์ถœํ•˜์—ฌ ํ…์ŠคํŠธ ํŒŒ์ผ ๋˜๋Š” stdout์œผ๋กœ ์ถœ๋ ฅํ•˜๋Š” `tools/extract_text.py` ๋„๊ตฌ๋ฅผ ์ถ”๊ฐ€ํ–ˆ์Šต๋‹ˆ๋‹ค. ### Security +- Bounded active parser work prevents one process from starting unbounded MinerU jobs under burst traffic. Excess requests fail before body reads with the RFC 6585 429 contract and RFC 9111 no-store semantics; gateway-level tenant quotas and durable job queues remain separate controls. - `/parse` authentication is now immutable per application instance and fails closed before multipart body parsing when required configuration is missing. Hostile missing, invalid, Unicode, oversized, and duplicated Authorization headers return one non-sensitive response. - Added unauthenticated `/ready` traffic readiness that combines authentication configuration with MinerU executable availability while `/health` remains liveness-only. - Hardened the Kubernetes deployment example with a restricted namespace policy, explicit non-root UID/GID, `RuntimeDefault` seccomp, disabled privilege escalation, dropped Linux capabilities, a read-only root filesystem, and bounded writable runtime volumes. @@ -64,7 +66,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- Improved OpenAPI metadata for the `/parse` endpoint by documenting 415, 502, and 503 error responses. +- Improved OpenAPI metadata for the `/parse` endpoint by documenting 415, 429, 502, and 503 error responses. ## [0.2.0] - 2026-04-24 From 090979e782d08278e538e5eed9c501f775692e73 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 13:59:26 +0900 Subject: [PATCH 45/75] docs(security): add parser resource-exhaustion controls --- docs/security/api-security-checklist.md | 48 +++++++++++++++++++------ 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/docs/security/api-security-checklist.md b/docs/security/api-security-checklist.md index 86be69fd..b670327e 100644 --- a/docs/security/api-security-checklist.md +++ b/docs/security/api-security-checklist.md @@ -5,6 +5,7 @@ Apply this checklist whenever the FastAPI surface changes. ## In-scope endpoints - `/health` +- `/ready` - `/parse` - `/docs` - `/redoc` @@ -12,18 +13,43 @@ Apply this checklist whenever the FastAPI surface changes. ## Baseline checks - validate upload handling and content-type expectations for `/parse` -- ensure error messages do not leak private reference paths, - secrets, or credentials -- keep synthetic fixtures in tests and examples; never use private - reference inputs in public evidence -- verify request handling fails safely when MinerU is missing or - returns incomplete outputs -- confirm docs endpoints remain informational and do not imply - unsupported authentication or execution guarantees +- ensure error messages do not leak private reference paths, secrets, + credentials, configured capacity, active lease counts, or another tenant's + activity +- keep synthetic fixtures in tests and examples; never use private reference + inputs in public evidence +- verify request handling fails safely when MinerU is missing or returns + incomplete outputs +- confirm `/health` is liveness-only and `/ready` represents authentication + configuration plus MinerU runtime availability +- authenticate `/parse` before admission so unauthenticated requests cannot + consume parser leases or use overload behavior as a capacity oracle +- acquire a non-waiting process-local lease before the multipart body, + temporary-file allocation, PDF validation, or MinerU execution +- return fixed `429 Too Many Requests`, exact `Retry-After: 1`, and no-store + cache headers when every process lease is in use +- release admission leases after success, validation failure, backend failure, + unhandled exception, and request cancellation +- keep `NEWSDOM_MAX_CONCURRENT_PARSES` immutable, bounded to `1..128`, and + explicit in production deployment examples +- treat the process boundary as a last-resort resource control; retain + gateway-level per-tenant rate limits, quotas, and cluster resource limits +- do not add an unbounded in-process request or upload queue +- confirm docs endpoints remain informational and do not imply unsupported + authentication, execution, queueing, or durability guarantees ## Verification expectations -- unit/integration tests for parsing and error handling -- live localhost smoke for `/health`, `/docs`, and `/redoc` when docs +- unit and integration tests for parsing, authentication, readiness, overload, + cancellation, and sanitized error handling +- a realistic concurrent burst proving active parser count never exceeds the + configured process budget and excess work never reaches multipart parsing +- OpenAPI evidence for 401, 413, 415, 422, 429, 502, and 503 responses +- live localhost smoke for `/health`, `/ready`, `/docs`, and `/redoc` when docs or screenshots change -- PR/workflow review whenever GitHub Actions, release, or code-scanning posture changes +- deployment smoke that holds every lease, verifies one fixed pre-body 429, + releases one request, and proves capacity recovery +- resource evidence for any capacity increase: peak RSS/VRAM, temporary storage, + parser latency, error rate, replica count, and rollback threshold +- PR and workflow review whenever GitHub Actions, release, or code-scanning + posture changes From 9fde28365ae3a77407df55f2a69b0cb36721ec50 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 14:03:50 +0900 Subject: [PATCH 46/75] test(benchmark): define upload-ingestion evidence contract --- tests/test_upload_ingestion_benchmark.py | 225 +++++++++++++++++++++++ 1 file changed, 225 insertions(+) create mode 100644 tests/test_upload_ingestion_benchmark.py diff --git a/tests/test_upload_ingestion_benchmark.py b/tests/test_upload_ingestion_benchmark.py new file mode 100644 index 00000000..672e4dc7 --- /dev/null +++ b/tests/test_upload_ingestion_benchmark.py @@ -0,0 +1,225 @@ +"""Contracts for the upload-ingestion benchmark and raw evidence schema.""" + +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +import pytest + +from tools import benchmark_upload_ingestion as benchmark + + +def test_candidate_matrix_preserves_every_issue_534_chunk_option() -> None: + """The harness must not silently omit or preselect one requested candidate.""" + + assert benchmark.CHUNK_CANDIDATES == { + "8kib": 8 * 1024, + "64kib": 64 * 1024, + "256kib": 256 * 1024, + "1mib": 1024 * 1024, + "adaptive": None, + } + assert benchmark.CONCURRENCY_LEVELS == (1, 8, 32, 128) + assert benchmark.TARGET_FIXTURE_BYTES == ( + 1 * 1024 * 1024, + 5 * 1024 * 1024, + 20 * 1024 * 1024, + ) + + +@pytest.mark.parametrize( + ("fixture_bytes", "expected"), + [ + (1 * 1024 * 1024, 64 * 1024), + (5 * 1024 * 1024, 256 * 1024), + (20 * 1024 * 1024, 1024 * 1024), + ], +) +def test_adaptive_candidate_resolves_deterministically( + fixture_bytes: int, + expected: int, +) -> None: + """The adaptive candidate needs an auditable size rule before measurement.""" + + assert benchmark.resolve_chunk_bytes("adaptive", fixture_bytes) == expected + + +def test_unknown_candidate_is_rejected() -> None: + """A typo must not fall back to an unrecorded upload policy.""" + + with pytest.raises(ValueError, match="Unknown chunk candidate"): + benchmark.resolve_chunk_bytes("unexpected", 1024) + + +def test_fixture_inventory_rejects_non_pdf_magic(tmp_path: Path) -> None: + """Raw benchmark evidence must identify actual PDF inputs.""" + + fixture = tmp_path / "invalid.pdf" + fixture.write_bytes(b"not a PDF") + + with pytest.raises(ValueError, match="PDF magic"): + benchmark.inventory_fixture(fixture) + + +def test_fixture_inventory_records_hash_and_size(tmp_path: Path) -> None: + """Every benchmark case should be traceable to immutable fixture bytes.""" + + fixture = tmp_path / "fixture.pdf" + fixture.write_bytes(b"%PDF-1.4\nsynthetic benchmark bytes\n%%EOF\n") + + record = benchmark.inventory_fixture(fixture) + + assert record["fixture_name"] == "fixture.pdf" + assert record["size_bytes"] == fixture.stat().st_size + assert record["sha256"].startswith("sha256:") + assert len(record["sha256"]) == len("sha256:") + 64 + + +def test_percentile_uses_nearest_rank_for_small_reproducible_samples() -> None: + """The raw report should define deterministic p50 and p95 aggregation.""" + + values = [0.4, 0.1, 0.3, 0.2] + + assert benchmark.nearest_rank_percentile(values, 0.50) == 0.2 + assert benchmark.nearest_rank_percentile(values, 0.95) == 0.4 + + +@pytest.mark.asyncio +async def test_matrix_report_records_environment_cases_and_raw_samples( + tmp_path: Path, +) -> None: + """A small injected run should preserve all inputs and unaggregated samples.""" + + fixture = tmp_path / "fixture.pdf" + fixture.write_bytes(b"%PDF-1.4\nsynthetic benchmark bytes\n%%EOF\n") + output = tmp_path / "report.json" + calls: list[tuple[str, int, int]] = [] + + async def fake_cohort( + fixture_path: Path, + *, + chunk_bytes: int, + concurrency: int, + ) -> list[dict[str, int | float]]: + calls.append((fixture_path.name, chunk_bytes, concurrency)) + return [ + { + "duration_seconds": 0.25, + "read_calls": 3, + "bytes_copied": fixture_path.stat().st_size, + } + for _ in range(concurrency) + ] + + report = await benchmark.run_matrix( + [fixture], + output_path=output, + candidates=("8kib", "adaptive"), + concurrency_levels=(1, 2), + repetitions=2, + cohort_runner=fake_cohort, + ) + + assert output.read_text(encoding="utf-8") == json.dumps( + report, + ensure_ascii=False, + indent=2, + sort_keys=True, + ) + "\n" + assert report["schema_version"] == "1.0.0" + assert report["benchmark_environment"]["python_version"] + assert len(report["fixtures"]) == 1 + assert len(report["cases"]) == 4 + assert len(report["cases"][0]["samples"]) == 2 + assert report["cases"][0]["metrics"]["p50_latency_seconds"] == 0.25 + assert report["cases"][0]["metrics"]["p95_latency_seconds"] == 0.25 + assert calls == [ + ("fixture.pdf", 8 * 1024, 1), + ("fixture.pdf", 8 * 1024, 1), + ("fixture.pdf", 64 * 1024, 1), + ("fixture.pdf", 64 * 1024, 1), + ("fixture.pdf", 8 * 1024, 2), + ("fixture.pdf", 8 * 1024, 2), + ("fixture.pdf", 64 * 1024, 2), + ("fixture.pdf", 64 * 1024, 2), + ] + + +def test_raw_evidence_schema_is_strict_and_covers_required_metrics() -> None: + """Checked-in evidence must be machine-auditable before results are accepted.""" + + schema_path = ( + Path(__file__).resolve().parents[1] + / "docs" + / "benchmarks" + / "upload-ingestion-result.schema.json" + ) + schema = json.loads(schema_path.read_text(encoding="utf-8")) + case_schema = schema["$defs"]["benchmark_case"] + metric_properties = case_schema["properties"]["metrics"]["properties"] + + assert schema["$schema"] == "https://json-schema.org/draft/2020-12/schema" + assert schema["additionalProperties"] is False + assert case_schema["additionalProperties"] is False + for required_metric in ( + "p50_latency_seconds", + "p95_latency_seconds", + "throughput_bytes_per_second", + "read_calls", + "cpu_seconds", + "peak_rss_bytes", + "temporary_disk_bytes", + "event_loop_max_delay_seconds", + ): + assert required_metric in metric_properties + assert required_metric in case_schema["properties"]["metrics"]["required"] + + +def test_cli_defaults_to_the_complete_evidence_matrix(monkeypatch, tmp_path: Path) -> None: + """The documented command should run every required candidate and concurrency.""" + + fixtures = tmp_path / "fixtures" + fixtures.mkdir() + output = tmp_path / "results.json" + captured: dict[str, object] = {} + + async def fake_run_matrix( + fixture_paths: list[Path], + *, + output_path: Path, + candidates: tuple[str, ...], + concurrency_levels: tuple[int, ...], + repetitions: int, + cohort_runner=benchmark.run_cohort, + ) -> dict[str, object]: + captured.update( + fixture_paths=fixture_paths, + output_path=output_path, + candidates=candidates, + concurrency_levels=concurrency_levels, + repetitions=repetitions, + cohort_runner=cohort_runner, + ) + return {} + + monkeypatch.setattr(benchmark, "run_matrix", fake_run_matrix) + monkeypatch.setattr(benchmark, "discover_fixtures", lambda _directory: []) + + benchmark.main( + [ + "--fixtures-dir", + str(fixtures), + "--output", + str(output), + "--repetitions", + "3", + ] + ) + + assert captured["candidates"] == tuple(benchmark.CHUNK_CANDIDATES) + assert captured["concurrency_levels"] == benchmark.CONCURRENCY_LEVELS + assert captured["repetitions"] == 3 + assert captured["output_path"] == output + assert asyncio.iscoroutinefunction(fake_run_matrix) From 8ac6a13c11dd94c72a8e85e6d895e8574b853b92 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 14:05:55 +0900 Subject: [PATCH 47/75] feat(benchmark): add reproducible upload-ingestion matrix --- tools/benchmark_upload_ingestion.py | 393 ++++++++++++++++++++++++++++ 1 file changed, 393 insertions(+) create mode 100644 tools/benchmark_upload_ingestion.py diff --git a/tools/benchmark_upload_ingestion.py b/tools/benchmark_upload_ingestion.py new file mode 100644 index 00000000..f04a6d3c --- /dev/null +++ b/tools/benchmark_upload_ingestion.py @@ -0,0 +1,393 @@ +"""Benchmark bounded PDF upload ingestion without selecting a new default.""" + +from __future__ import annotations + +import argparse +import asyncio +import hashlib +import json +import math +import os +import platform +import sys +import tempfile +import time +from collections.abc import Awaitable, Callable, Sequence +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from starlette.datastructures import UploadFile + +CHUNK_CANDIDATES: dict[str, int | None] = { + "8kib": 8 * 1024, + "64kib": 64 * 1024, + "256kib": 256 * 1024, + "1mib": 1024 * 1024, + "adaptive": None, +} +CONCURRENCY_LEVELS = (1, 8, 32, 128) +TARGET_FIXTURE_BYTES = ( + 1 * 1024 * 1024, + 5 * 1024 * 1024, + 20 * 1024 * 1024, +) +_RESULT_SCHEMA_PATH = "docs/benchmarks/upload-ingestion-result.schema.json" +CohortRunner = Callable[..., Awaitable[list[dict[str, int | float]]]] + + +def resolve_chunk_bytes(candidate: str, fixture_bytes: int) -> int: + """Resolve one named candidate to the exact read size used for a fixture.""" + + if candidate not in CHUNK_CANDIDATES: + raise ValueError(f"Unknown chunk candidate: {candidate}") + configured = CHUNK_CANDIDATES[candidate] + if configured is not None: + return configured + if fixture_bytes <= TARGET_FIXTURE_BYTES[0]: + return 64 * 1024 + if fixture_bytes <= TARGET_FIXTURE_BYTES[1]: + return 256 * 1024 + return 1024 * 1024 + + +def _sha256(path: Path) -> str: + """Return a streaming SHA-256 identifier for one fixture.""" + + digest = hashlib.sha256() + with path.open("rb") as handle: + while chunk := handle.read(1024 * 1024): + digest.update(chunk) + return f"sha256:{digest.hexdigest()}" + + +def inventory_fixture(path: Path) -> dict[str, int | str]: + """Validate and inventory one immutable PDF benchmark fixture.""" + + if not path.is_file() or path.is_symlink(): + raise ValueError(f"Fixture must be a regular non-symlink file: {path}") + with path.open("rb") as handle: + if handle.read(5) != b"%PDF-": + raise ValueError(f"Fixture does not contain PDF magic bytes: {path.name}") + return { + "fixture_name": path.name, + "size_bytes": path.stat().st_size, + "sha256": _sha256(path), + } + + +def discover_fixtures(directory: Path) -> list[Path]: + """Return the sorted regular PDF fixtures directly under a directory.""" + + if not directory.is_dir(): + raise ValueError(f"Fixture directory does not exist: {directory}") + return sorted( + path + for path in directory.glob("*.pdf") + if path.is_file() and not path.is_symlink() + ) + + +def nearest_rank_percentile(values: Sequence[float], probability: float) -> float: + """Return the deterministic nearest-rank percentile for non-empty samples.""" + + if not values: + raise ValueError("Percentile requires at least one sample") + if not 0 < probability <= 1: + raise ValueError("Percentile probability must be in (0, 1]") + ordered = sorted(values) + rank = max(1, math.ceil(probability * len(ordered))) + return ordered[rank - 1] + + +def _peak_rss_bytes() -> int: + """Return the process peak RSS in bytes when the platform exposes it.""" + + try: + import resource + except ImportError: + return 0 + raw_value = int(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss) + if platform.system() == "Darwin": + return raw_value + return raw_value * 1024 + + +async def _event_loop_probe( + stop_event: asyncio.Event, + observed_delays: list[float], + *, + interval_seconds: float = 0.005, +) -> None: + """Measure maximum event-loop scheduling delay while a cohort is active.""" + + expected = time.perf_counter() + interval_seconds + while not stop_event.is_set(): + await asyncio.sleep(interval_seconds) + now = time.perf_counter() + observed_delays.append(max(0.0, now - expected)) + expected = now + interval_seconds + + +async def _copy_one_fixture( + fixture_path: Path, + *, + chunk_bytes: int, + temporary_directory: Path, +) -> dict[str, int | float]: + """Copy one real file-backed UploadFile to temporary storage and measure it.""" + + started = time.perf_counter() + read_calls = 0 + bytes_copied = 0 + source_handle = fixture_path.open("rb") + upload = UploadFile( + file=source_handle, + size=fixture_path.stat().st_size, + filename=fixture_path.name, + ) + try: + with tempfile.NamedTemporaryFile( + dir=temporary_directory, + prefix="newsdom-ingestion-benchmark-", + delete=True, + ) as destination: + while True: + chunk = await upload.read(chunk_bytes) + read_calls += 1 + if not chunk: + break + destination.write(chunk) + bytes_copied += len(chunk) + destination.flush() + finally: + await upload.close() + return { + "duration_seconds": time.perf_counter() - started, + "read_calls": read_calls, + "bytes_copied": bytes_copied, + } + + +async def run_cohort( + fixture_path: Path, + *, + chunk_bytes: int, + concurrency: int, +) -> list[dict[str, int | float]]: + """Run one concurrent real-file cohort and preserve per-request raw samples.""" + + if concurrency < 1: + raise ValueError("Concurrency must be at least one") + if chunk_bytes < 1: + raise ValueError("Chunk size must be at least one byte") + + stop_event = asyncio.Event() + observed_delays: list[float] = [] + with tempfile.TemporaryDirectory(prefix="newsdom-ingestion-benchmark-") as root: + temporary_directory = Path(root) + probe = asyncio.create_task(_event_loop_probe(stop_event, observed_delays)) + try: + samples = await asyncio.gather( + *( + _copy_one_fixture( + fixture_path, + chunk_bytes=chunk_bytes, + temporary_directory=temporary_directory, + ) + for _ in range(concurrency) + ) + ) + finally: + stop_event.set() + await probe + + maximum_delay = max(observed_delays, default=0.0) + for sample in samples: + sample["event_loop_max_delay_seconds"] = maximum_delay + return samples + + +def _environment_record() -> dict[str, Any]: + """Return reproducibility metadata that does not contain credentials.""" + + return { + "python_version": platform.python_version(), + "python_implementation": platform.python_implementation(), + "executable": Path(sys.executable).name, + "platform": platform.platform(), + "machine": platform.machine(), + "processor_count": os.cpu_count(), + } + + +def _metrics( + samples: Sequence[dict[str, int | float]], + *, + cohort_wall_seconds: Sequence[float], + cpu_seconds: Sequence[float], + peak_rss_bytes: Sequence[int], +) -> dict[str, int | float]: + """Aggregate raw samples without discarding the samples themselves.""" + + durations = [float(sample["duration_seconds"]) for sample in samples] + total_bytes = sum(int(sample["bytes_copied"]) for sample in samples) + total_wall = sum(cohort_wall_seconds) + return { + "p50_latency_seconds": nearest_rank_percentile(durations, 0.50), + "p95_latency_seconds": nearest_rank_percentile(durations, 0.95), + "throughput_bytes_per_second": total_bytes / total_wall if total_wall else 0.0, + "read_calls": sum(int(sample["read_calls"]) for sample in samples), + "cpu_seconds": sum(cpu_seconds), + "peak_rss_bytes": max(peak_rss_bytes, default=0), + "temporary_disk_bytes": total_bytes, + "event_loop_max_delay_seconds": max( + ( + float(sample.get("event_loop_max_delay_seconds", 0.0)) + for sample in samples + ), + default=0.0, + ), + } + + +async def run_matrix( + fixture_paths: list[Path], + *, + output_path: Path, + candidates: tuple[str, ...] = tuple(CHUNK_CANDIDATES), + concurrency_levels: tuple[int, ...] = CONCURRENCY_LEVELS, + repetitions: int = 3, + cohort_runner: CohortRunner = run_cohort, +) -> dict[str, Any]: + """Run and write the reproducible raw upload-ingestion evidence matrix.""" + + if repetitions < 1: + raise ValueError("Repetitions must be at least one") + if not fixture_paths: + raise ValueError("At least one PDF fixture is required") + + fixture_records = [inventory_fixture(path) for path in fixture_paths] + cases: list[dict[str, Any]] = [] + for concurrency in concurrency_levels: + if concurrency < 1: + raise ValueError("Concurrency levels must be positive") + for fixture_path, fixture_record in zip(fixture_paths, fixture_records): + fixture_bytes = int(fixture_record["size_bytes"]) + for candidate in candidates: + chunk_bytes = resolve_chunk_bytes(candidate, fixture_bytes) + all_samples: list[dict[str, int | float]] = [] + cohort_wall_seconds: list[float] = [] + cpu_seconds: list[float] = [] + peak_rss_bytes: list[int] = [] + for _ in range(repetitions): + wall_started = time.perf_counter() + cpu_started = time.process_time() + samples = await cohort_runner( + fixture_path, + chunk_bytes=chunk_bytes, + concurrency=concurrency, + ) + cpu_seconds.append(time.process_time() - cpu_started) + cohort_wall_seconds.append(time.perf_counter() - wall_started) + peak_rss_bytes.append(_peak_rss_bytes()) + all_samples.extend(samples) + cases.append( + { + "fixture_name": fixture_path.name, + "candidate": candidate, + "chunk_bytes": chunk_bytes, + "concurrency": concurrency, + "repetitions": repetitions, + "metrics": _metrics( + all_samples, + cohort_wall_seconds=cohort_wall_seconds, + cpu_seconds=cpu_seconds, + peak_rss_bytes=peak_rss_bytes, + ), + "samples": all_samples, + } + ) + + report: dict[str, Any] = { + "schema_version": "1.0.0", + "schema_path": _RESULT_SCHEMA_PATH, + "generated_at": datetime.now(timezone.utc).isoformat(), + "benchmark_environment": _environment_record(), + "fixtures": fixture_records, + "cases": cases, + } + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text( + json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return report + + +def _positive_int(value: str) -> int: + """Parse one strictly positive command-line integer.""" + + parsed = int(value) + if parsed < 1: + raise argparse.ArgumentTypeError("value must be at least one") + return parsed + + +def main(argv: list[str] | None = None) -> None: + """Run the complete issue #534 candidate matrix from the command line.""" + + parser = argparse.ArgumentParser( + description=( + "Benchmark PDF UploadFile ingestion candidates and emit raw JSON evidence." + ) + ) + parser.add_argument( + "--fixtures-dir", + type=Path, + required=True, + help="Directory containing real 1 MiB, 5 MiB, and 20 MiB PDF fixtures.", + ) + parser.add_argument( + "--output", + type=Path, + required=True, + help="Path for the schema-versioned raw JSON report.", + ) + parser.add_argument( + "--repetitions", + type=_positive_int, + default=3, + help="Cohort repetitions for every matrix case (default: 3).", + ) + parser.add_argument( + "--candidate", + dest="candidates", + action="append", + choices=tuple(CHUNK_CANDIDATES), + help="Optional candidate filter; repeat to select multiple values.", + ) + parser.add_argument( + "--concurrency", + dest="concurrency_levels", + action="append", + type=_positive_int, + help="Optional concurrency filter; repeat to select multiple values.", + ) + args = parser.parse_args(argv) + fixture_paths = discover_fixtures(args.fixtures_dir) + candidates = tuple(args.candidates or CHUNK_CANDIDATES) + concurrency_levels = tuple(args.concurrency_levels or CONCURRENCY_LEVELS) + asyncio.run( + run_matrix( + fixture_paths, + output_path=args.output, + candidates=candidates, + concurrency_levels=concurrency_levels, + repetitions=args.repetitions, + ) + ) + + +if __name__ == "__main__": # pragma: no cover + main() From 3b516dceb27c94930c707b3ff49d6b8dd10976fe Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 14:06:49 +0900 Subject: [PATCH 48/75] docs(benchmark): add strict upload-ingestion evidence schema --- .../upload-ingestion-result.schema.json | 218 ++++++++++++++++++ 1 file changed, 218 insertions(+) create mode 100644 docs/benchmarks/upload-ingestion-result.schema.json diff --git a/docs/benchmarks/upload-ingestion-result.schema.json b/docs/benchmarks/upload-ingestion-result.schema.json new file mode 100644 index 00000000..91761db1 --- /dev/null +++ b/docs/benchmarks/upload-ingestion-result.schema.json @@ -0,0 +1,218 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/ContextualWisdomLab/newsdom-api/blob/develop/docs/benchmarks/upload-ingestion-result.schema.json", + "title": "NewsDOM upload ingestion benchmark result", + "description": "Raw, reproducible evidence for issue #534 chunk-size and concurrency candidates.", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "schema_path", + "generated_at", + "benchmark_environment", + "fixtures", + "cases" + ], + "properties": { + "schema_version": { + "const": "1.0.0" + }, + "schema_path": { + "const": "docs/benchmarks/upload-ingestion-result.schema.json" + }, + "generated_at": { + "type": "string", + "format": "date-time" + }, + "benchmark_environment": { + "$ref": "#/$defs/benchmark_environment" + }, + "fixtures": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/fixture_record" + } + }, + "cases": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/benchmark_case" + } + } + }, + "$defs": { + "benchmark_environment": { + "type": "object", + "additionalProperties": false, + "required": [ + "python_version", + "python_implementation", + "executable", + "platform", + "machine", + "processor_count" + ], + "properties": { + "python_version": { + "type": "string", + "minLength": 1 + }, + "python_implementation": { + "type": "string", + "minLength": 1 + }, + "executable": { + "type": "string", + "minLength": 1 + }, + "platform": { + "type": "string", + "minLength": 1 + }, + "machine": { + "type": "string" + }, + "processor_count": { + "type": ["integer", "null"], + "minimum": 1 + } + } + }, + "fixture_record": { + "type": "object", + "additionalProperties": false, + "required": ["fixture_name", "size_bytes", "sha256"], + "properties": { + "fixture_name": { + "type": "string", + "minLength": 1 + }, + "size_bytes": { + "type": "integer", + "minimum": 1, + "maximum": 20971520 + }, + "sha256": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + } + } + }, + "benchmark_case": { + "type": "object", + "additionalProperties": false, + "required": [ + "fixture_name", + "candidate", + "chunk_bytes", + "concurrency", + "repetitions", + "metrics", + "samples" + ], + "properties": { + "fixture_name": { + "type": "string", + "minLength": 1 + }, + "candidate": { + "enum": ["8kib", "64kib", "256kib", "1mib", "adaptive"] + }, + "chunk_bytes": { + "type": "integer", + "minimum": 8192, + "maximum": 1048576 + }, + "concurrency": { + "type": "integer", + "enum": [1, 8, 32, 128] + }, + "repetitions": { + "type": "integer", + "minimum": 1 + }, + "metrics": { + "type": "object", + "additionalProperties": false, + "required": [ + "p50_latency_seconds", + "p95_latency_seconds", + "throughput_bytes_per_second", + "read_calls", + "cpu_seconds", + "peak_rss_bytes", + "temporary_disk_bytes", + "event_loop_max_delay_seconds" + ], + "properties": { + "p50_latency_seconds": { + "type": "number", + "minimum": 0 + }, + "p95_latency_seconds": { + "type": "number", + "minimum": 0 + }, + "throughput_bytes_per_second": { + "type": "number", + "minimum": 0 + }, + "read_calls": { + "type": "integer", + "minimum": 1 + }, + "cpu_seconds": { + "type": "number", + "minimum": 0 + }, + "peak_rss_bytes": { + "type": "integer", + "minimum": 0 + }, + "temporary_disk_bytes": { + "type": "integer", + "minimum": 1 + }, + "event_loop_max_delay_seconds": { + "type": "number", + "minimum": 0 + } + } + }, + "samples": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/sample_record" + } + } + } + }, + "sample_record": { + "type": "object", + "additionalProperties": false, + "required": ["duration_seconds", "read_calls", "bytes_copied"], + "properties": { + "duration_seconds": { + "type": "number", + "minimum": 0 + }, + "read_calls": { + "type": "integer", + "minimum": 1 + }, + "bytes_copied": { + "type": "integer", + "minimum": 1, + "maximum": 20971520 + }, + "event_loop_max_delay_seconds": { + "type": "number", + "minimum": 0 + } + } + } + } +} From 968e2b406b4f3ee07a5286c2fbe1f404e7268ab2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 14:09:23 +0900 Subject: [PATCH 49/75] docs(benchmark): record upload measurement methodology --- .../upload-ingestion-benchmark-methodology.md | 161 ++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 docs/doctoring/upload-ingestion-benchmark-methodology.md diff --git a/docs/doctoring/upload-ingestion-benchmark-methodology.md b/docs/doctoring/upload-ingestion-benchmark-methodology.md new file mode 100644 index 00000000..2a9037ef --- /dev/null +++ b/docs/doctoring/upload-ingestion-benchmark-methodology.md @@ -0,0 +1,161 @@ +# Upload-ingestion benchmark methodology + +## Purpose + +This record defines the reproducible evidence required before NewsDOM changes +the current 8 KiB upload read size. It deliberately does **not** select a new +default or claim a performance improvement. The benchmark compares every +candidate required by issue #534 while the production parser admission boundary +remains fixed and independently testable. + +FastAPI documents that the asynchronous `UploadFile` methods run the underlying +file operations in a thread pool. Starlette documents that `UploadFile` shares +AnyIO's thread-pool capacity with synchronous endpoints, file responses, +background tasks, and other framework work. Chunk size therefore affects more +than sequential file-copy throughput: it changes the number of thread-pool +crossings, event-loop responsiveness, and the amount of data retained by each +concurrent request. + +## Candidate matrix + +The default command measures the following exact candidates: + +| Candidate | Exact or resolved read size | +| --- | ---: | +| `8kib` | 8,192 bytes | +| `64kib` | 65,536 bytes | +| `256kib` | 262,144 bytes | +| `1mib` | 1,048,576 bytes | +| `adaptive` | 64 KiB at 1 MiB, 256 KiB through 5 MiB, otherwise 1 MiB | + +Every candidate is measured at concurrency `1`, `8`, `32`, and `128`. The +fixture set must contain representative PDFs near 1 MiB, 5 MiB, and the existing +20 MiB request limit. Fixtures remain outside public git when licensing, +privacy, or contractual restrictions apply. The raw report stores each fixture +name, byte size, and SHA-256 identifier so an internal evidence bundle can be +audited without publishing document content. + +The adaptive rule is only a benchmark candidate. It is not production behavior +and must not be described as selected until a reviewed result satisfies the +acceptance rules below. + +## Reproducible command + +```bash +uv run python tools/benchmark_upload_ingestion.py \ + --fixtures-dir /absolute/path/to/approved-pdf-fixtures \ + --output artifacts/upload-ingestion-result.json \ + --repetitions 3 +``` + +The default command runs the complete candidate and concurrency matrix. Filters +are available for diagnosis, but a filtered run is not sufficient for a default +selection. Execute the benchmark on an otherwise idle, dedicated environment +and record the container or host image, CPU model, memory, storage class, +filesystem, Python and dependency lock, commit SHA, worker count, and whether +the files were cold or warm in the operating-system cache. + +## Measurement implementation + +`tools/benchmark_upload_ingestion.py` uses a real file-backed Starlette +`UploadFile` and copies each request to an independent temporary file. It does +not use an in-memory bytes object as the source of accepted evidence. + +The harness records unaggregated samples and derives: + +- p50 and p95 request latency by deterministic nearest-rank percentile; +- aggregate copied bytes divided by cohort wall time; +- total `UploadFile.read(size)` calls; +- process CPU time; +- process peak resident set size when the platform exposes `getrusage`; +- temporary bytes written; +- maximum observed event-loop scheduling delay. + +Python defines `time.perf_counter()` as a high-resolution clock suitable for +short durations and `time.process_time()` as process CPU time excluding sleep. +Python's `resource.getrusage()` exposes `ru_maxrss`, but the unit is platform +dependent. The harness normalizes Linux values from KiB and Darwin values from +bytes. Windows reports zero for peak RSS in this version and cannot serve as the +accepted memory-evidence environment without an additional reviewed provider. + +Peak RSS is a process high-water mark rather than an isolated per-case delta. +Accordingly, execute candidates in separate fresh processes when making the +final memory decision, or treat later in-process values as conservative upper +bounds. Do not subtract two high-water marks and call the result allocated +memory. + +## Raw evidence schema + +The checked-in JSON Schema is: + +```text +docs/benchmarks/upload-ingestion-result.schema.json +``` + +Accepted evidence must validate against schema version `1.0.0`. The report keeps +raw samples alongside aggregates, rejects unknown top-level and case fields, +and records the exact resolved byte size even for the adaptive candidate. +Generated reports are evidence artifacts, not source files; do not commit +private fixture paths or customer document names. + +## Interpretation and selection gates + +A candidate can replace 8 KiB only when the exact current-head report shows all +of the following across the complete matrix: + +1. no upload-size, magic-byte, temporary-file cleanup, authentication, or parser + admission regression; +2. no materially worse p95 latency or event-loop delay at high concurrency; +3. peak RSS and temporary storage remain inside an explicit per-process budget; +4. thread-pool read-call reduction does not merely shift cost to larger retained + buffers or storage stalls; +5. results are reproduced on at least two runs from clean processes; +6. the selected rule is simple enough to explain, test, operate, and roll back; +7. the production regression test, README, architecture, runbook, CHANGELOG, and + release evidence all identify the same value or rule. + +Compare distributions and operational thresholds rather than publishing a +single mean. A percentage improvement may be stated only when the checked-in or +attached raw report, environment manifest, analysis code, and confidence or +variability treatment support it. + +## Accuracy and security invariants + +Chunk experiments must preserve: + +- authentication and process admission before multipart body consumption; +- first-five-byte PDF magic rejection; +- the exact 20 MiB streamed limit even when size metadata is missing; +- no write of the chunk that crosses the limit; +- request-scoped temporary files and cleanup after every outcome; +- structural PDF validation before MinerU; +- parser-not-called evidence for rejected requests; +- fixed sanitized errors and no-store response headers. + +The benchmark is not permission to bypass `NEWSDOM_MAX_CONCURRENT_PARSES`. +Concurrency cases measure ingestion behavior under controlled experiments; a +production deployment must continue to reject active parser work above its +accepted process budget. + +## Rollback + +Keep 8 KiB as the rollback value until a new policy has exact-head benchmark and +release acceptance. If a deployed policy increases RSS, event-loop delay, +temporary-storage pressure, timeout rate, or parser failures, restore the prior +value or release, restart processes so immutable configuration and code are +loaded, and rerun the overload and upload-limit smoke tests. + +## References + +FastAPI. (2026). *Request files*. FastAPI documentation. +https://fastapi.tiangolo.com/tutorial/request-files/ + +Python Software Foundation. (2026). *resourceโ€”Resource usage information +(Python 3.10.20 documentation)*. +https://docs.python.org/3.10/library/resource.html + +Python Software Foundation. (2026). *timeโ€”Time access and conversions (Python +3.10.20 documentation)*. https://docs.python.org/3.10/library/time.html + +Starlette. (2026). *Thread pool*. Starlette documentation. +https://www.starlette.io/threadpool/ From 8995dd82fe1217c96e2cdb160ee7b750f9480c22 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 14:16:23 +0900 Subject: [PATCH 50/75] test(benchmark): allow audited custom concurrency reports --- tests/test_upload_ingestion_benchmark.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_upload_ingestion_benchmark.py b/tests/test_upload_ingestion_benchmark.py index 672e4dc7..5df9a2ba 100644 --- a/tests/test_upload_ingestion_benchmark.py +++ b/tests/test_upload_ingestion_benchmark.py @@ -163,6 +163,10 @@ def test_raw_evidence_schema_is_strict_and_covers_required_metrics() -> None: assert schema["$schema"] == "https://json-schema.org/draft/2020-12/schema" assert schema["additionalProperties"] is False assert case_schema["additionalProperties"] is False + assert case_schema["properties"]["concurrency"] == { + "type": "integer", + "minimum": 1, + } for required_metric in ( "p50_latency_seconds", "p95_latency_seconds", From b30686164710012494b6ed9857e6ab6f87e96aa5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 6 Aug 2026 14:17:26 +0900 Subject: [PATCH 51/75] fix(benchmark): validate custom positive concurrency --- docs/benchmarks/upload-ingestion-result.schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/benchmarks/upload-ingestion-result.schema.json b/docs/benchmarks/upload-ingestion-result.schema.json index 91761db1..951e2709 100644 --- a/docs/benchmarks/upload-ingestion-result.schema.json +++ b/docs/benchmarks/upload-ingestion-result.schema.json @@ -127,7 +127,7 @@ }, "concurrency": { "type": "integer", - "enum": [1, 8, 32, 128] + "minimum": 1 }, "repetitions": { "type": "integer", From c4fbda64790d4920e98df638296f90c8c46d5d15 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 7 Aug 2026 08:49:09 +0900 Subject: [PATCH 52/75] test(benchmark): require case-local RSS and true event-loop p95 evidence --- tests/test_upload_ingestion_benchmark.py | 125 +++++++++++++++++++++-- 1 file changed, 115 insertions(+), 10 deletions(-) diff --git a/tests/test_upload_ingestion_benchmark.py b/tests/test_upload_ingestion_benchmark.py index 5df9a2ba..f67fc44f 100644 --- a/tests/test_upload_ingestion_benchmark.py +++ b/tests/test_upload_ingestion_benchmark.py @@ -11,6 +11,21 @@ from tools import benchmark_upload_ingestion as benchmark +def _sample( + duration: float = 0.25, + *, + read_calls: int = 3, + bytes_copied: int = 100, +) -> dict[str, int | float]: + """Return one compact request sample for aggregate-metric tests.""" + + return { + "duration_seconds": duration, + "read_calls": read_calls, + "bytes_copied": bytes_copied, + } + + def test_candidate_matrix_preserves_every_issue_534_chunk_option() -> None: """The harness must not silently omit or preselect one requested candidate.""" @@ -86,6 +101,82 @@ def test_percentile_uses_nearest_rank_for_small_reproducible_samples() -> None: assert benchmark.nearest_rank_percentile(values, 0.95) == 0.4 +def test_current_rss_reads_linux_vmrss_in_bytes(tmp_path: Path) -> None: + """Per-case memory evidence must use current RSS rather than lifetime high-water.""" + + status_path = tmp_path / "status" + status_path.write_text("Name:\tpython\nVmRSS:\t1234 kB\n", encoding="utf-8") + + assert benchmark.current_rss_bytes(status_path=status_path) == 1234 * 1024 + + +def test_current_rss_returns_zero_when_status_is_missing_or_malformed( + tmp_path: Path, +) -> None: + """Unsupported memory providers should be explicit zero evidence, not stale RSS.""" + + missing = tmp_path / "missing" + malformed = tmp_path / "status" + malformed.write_text("VmRSS:\tnot-a-number kB\n", encoding="utf-8") + + assert benchmark.current_rss_bytes(status_path=missing) == 0 + assert benchmark.current_rss_bytes(status_path=malformed) == 0 + + +def test_metrics_use_case_local_rss_and_nearest_rank_event_loop_p95() -> None: + """Memory and responsiveness aggregates must derive from raw cohort evidence.""" + + first = benchmark.CohortObservation( + samples=(_sample(0.10), _sample(0.30)), + wall_seconds=0.40, + cpu_seconds=0.20, + rss_baseline_bytes=100, + peak_rss_bytes=500, + event_loop_delay_samples_seconds=(0.001, 0.002, 0.003, 0.100), + ) + second = benchmark.CohortObservation( + samples=(_sample(0.20),), + wall_seconds=0.20, + cpu_seconds=0.10, + rss_baseline_bytes=150, + peak_rss_bytes=250, + event_loop_delay_samples_seconds=(0.004, 0.005), + ) + + metrics = benchmark.aggregate_metrics((first, second)) + + assert metrics["peak_rss_bytes"] == 500 + assert metrics["peak_rss_delta_bytes"] == 400 + assert metrics["event_loop_p95_delay_seconds"] == 0.100 + assert metrics["event_loop_max_delay_seconds"] == 0.100 + assert metrics["p50_latency_seconds"] == 0.20 + assert metrics["p95_latency_seconds"] == 0.30 + + +def test_later_case_can_report_lower_rss_than_an_earlier_case() -> None: + """A process-lifetime high-water mark must not contaminate later matrix cases.""" + + high = benchmark.CohortObservation( + samples=(_sample(),), + wall_seconds=0.25, + cpu_seconds=0.10, + rss_baseline_bytes=1_000, + peak_rss_bytes=9_000, + event_loop_delay_samples_seconds=(), + ) + low = benchmark.CohortObservation( + samples=(_sample(),), + wall_seconds=0.25, + cpu_seconds=0.10, + rss_baseline_bytes=1_000, + peak_rss_bytes=2_000, + event_loop_delay_samples_seconds=(), + ) + + assert benchmark.aggregate_metrics((high,))["peak_rss_bytes"] == 9_000 + assert benchmark.aggregate_metrics((low,))["peak_rss_bytes"] == 2_000 + + @pytest.mark.asyncio async def test_matrix_report_records_environment_cases_and_raw_samples( tmp_path: Path, @@ -102,16 +193,23 @@ async def fake_cohort( *, chunk_bytes: int, concurrency: int, - ) -> list[dict[str, int | float]]: + ) -> benchmark.CohortObservation: calls.append((fixture_path.name, chunk_bytes, concurrency)) - return [ - { - "duration_seconds": 0.25, - "read_calls": 3, - "bytes_copied": fixture_path.stat().st_size, - } - for _ in range(concurrency) - ] + return benchmark.CohortObservation( + samples=tuple( + { + "duration_seconds": 0.25, + "read_calls": 3, + "bytes_copied": fixture_path.stat().st_size, + } + for _ in range(concurrency) + ), + wall_seconds=0.50, + cpu_seconds=0.25, + rss_baseline_bytes=1_000, + peak_rss_bytes=2_000, + event_loop_delay_samples_seconds=(0.001, 0.010), + ) report = await benchmark.run_matrix( [fixture], @@ -128,13 +226,16 @@ async def fake_cohort( indent=2, sort_keys=True, ) + "\n" - assert report["schema_version"] == "1.0.0" + assert report["schema_version"] == "1.1.0" assert report["benchmark_environment"]["python_version"] assert len(report["fixtures"]) == 1 assert len(report["cases"]) == 4 assert len(report["cases"][0]["samples"]) == 2 + assert len(report["cases"][0]["cohort_observations"]) == 2 assert report["cases"][0]["metrics"]["p50_latency_seconds"] == 0.25 assert report["cases"][0]["metrics"]["p95_latency_seconds"] == 0.25 + assert report["cases"][0]["metrics"]["peak_rss_delta_bytes"] == 1_000 + assert report["cases"][0]["metrics"]["event_loop_p95_delay_seconds"] == 0.010 assert calls == [ ("fixture.pdf", 8 * 1024, 1), ("fixture.pdf", 8 * 1024, 1), @@ -161,8 +262,10 @@ def test_raw_evidence_schema_is_strict_and_covers_required_metrics() -> None: metric_properties = case_schema["properties"]["metrics"]["properties"] assert schema["$schema"] == "https://json-schema.org/draft/2020-12/schema" + assert schema["properties"]["schema_version"] == {"const": "1.1.0"} assert schema["additionalProperties"] is False assert case_schema["additionalProperties"] is False + assert "cohort_observations" in case_schema["required"] assert case_schema["properties"]["concurrency"] == { "type": "integer", "minimum": 1, @@ -174,7 +277,9 @@ def test_raw_evidence_schema_is_strict_and_covers_required_metrics() -> None: "read_calls", "cpu_seconds", "peak_rss_bytes", + "peak_rss_delta_bytes", "temporary_disk_bytes", + "event_loop_p95_delay_seconds", "event_loop_max_delay_seconds", ): assert required_metric in metric_properties From eb0e2d304f7a2dc59a26a5cca94f005f9a72c82a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 7 Aug 2026 08:51:19 +0900 Subject: [PATCH 53/75] fix(benchmark): measure case-local RSS and event-loop p95 --- tools/benchmark_upload_ingestion.py | 183 ++++++++++++++++++++-------- 1 file changed, 129 insertions(+), 54 deletions(-) diff --git a/tools/benchmark_upload_ingestion.py b/tools/benchmark_upload_ingestion.py index f04a6d3c..487869c0 100644 --- a/tools/benchmark_upload_ingestion.py +++ b/tools/benchmark_upload_ingestion.py @@ -13,6 +13,7 @@ import tempfile import time from collections.abc import Awaitable, Callable, Sequence +from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path from typing import Any @@ -33,7 +34,39 @@ 20 * 1024 * 1024, ) _RESULT_SCHEMA_PATH = "docs/benchmarks/upload-ingestion-result.schema.json" -CohortRunner = Callable[..., Awaitable[list[dict[str, int | float]]]] +SampleRecord = dict[str, int | float] + + +@dataclass(frozen=True, slots=True) +class CohortObservation: + """Raw process and request evidence for one concurrent ingestion cohort.""" + + samples: tuple[SampleRecord, ...] + wall_seconds: float + cpu_seconds: float + rss_baseline_bytes: int + peak_rss_bytes: int + event_loop_delay_samples_seconds: tuple[float, ...] + + def as_record(self) -> dict[str, Any]: + """Return the JSON-serializable cohort evidence retained in the report.""" + + return { + "wall_seconds": self.wall_seconds, + "cpu_seconds": self.cpu_seconds, + "rss_baseline_bytes": self.rss_baseline_bytes, + "peak_rss_bytes": self.peak_rss_bytes, + "peak_rss_delta_bytes": max( + 0, + self.peak_rss_bytes - self.rss_baseline_bytes, + ), + "event_loop_delay_samples_seconds": list( + self.event_loop_delay_samples_seconds + ), + } + + +CohortRunner = Callable[..., Awaitable[CohortObservation]] def resolve_chunk_bytes(candidate: str, fixture_bytes: int) -> int: @@ -100,17 +133,25 @@ def nearest_rank_percentile(values: Sequence[float], probability: float) -> floa return ordered[rank - 1] -def _peak_rss_bytes() -> int: - """Return the process peak RSS in bytes when the platform exposes it.""" +def current_rss_bytes(*, status_path: Path = Path("/proc/self/status")) -> int: + """Return current Linux process RSS in bytes, or zero when unavailable.""" try: - import resource - except ImportError: + lines = status_path.read_text(encoding="utf-8").splitlines() + except OSError: return 0 - raw_value = int(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss) - if platform.system() == "Darwin": - return raw_value - return raw_value * 1024 + for line in lines: + if not line.startswith("VmRSS:"): + continue + fields = line.split() + if len(fields) != 3 or fields[2] != "kB": + return 0 + try: + kibibytes = int(fields[1]) + except ValueError: + return 0 + return max(0, kibibytes) * 1024 + return 0 async def _event_loop_probe( @@ -119,7 +160,7 @@ async def _event_loop_probe( *, interval_seconds: float = 0.005, ) -> None: - """Measure maximum event-loop scheduling delay while a cohort is active.""" + """Sample event-loop scheduling delays while one cohort is active.""" expected = time.perf_counter() + interval_seconds while not stop_event.is_set(): @@ -129,12 +170,27 @@ async def _event_loop_probe( expected = now + interval_seconds +async def _rss_probe( + stop_event: asyncio.Event, + observed_rss_bytes: list[int], + *, + rss_reader: Callable[[], int] = current_rss_bytes, + interval_seconds: float = 0.005, +) -> None: + """Sample current process RSS so each case has an independent peak.""" + + observed_rss_bytes.append(rss_reader()) + while not stop_event.is_set(): + await asyncio.sleep(interval_seconds) + observed_rss_bytes.append(rss_reader()) + + async def _copy_one_fixture( fixture_path: Path, *, chunk_bytes: int, temporary_directory: Path, -) -> dict[str, int | float]: +) -> SampleRecord: """Copy one real file-backed UploadFile to temporary storage and measure it.""" started = time.perf_counter() @@ -174,19 +230,26 @@ async def run_cohort( *, chunk_bytes: int, concurrency: int, -) -> list[dict[str, int | float]]: - """Run one concurrent real-file cohort and preserve per-request raw samples.""" +) -> CohortObservation: + """Run one real-file cohort and retain raw timing, loop, and RSS evidence.""" if concurrency < 1: raise ValueError("Concurrency must be at least one") if chunk_bytes < 1: raise ValueError("Chunk size must be at least one byte") + wall_started = time.perf_counter() + cpu_started = time.process_time() + rss_baseline_bytes = current_rss_bytes() + observed_rss_bytes = [rss_baseline_bytes] stop_event = asyncio.Event() observed_delays: list[float] = [] with tempfile.TemporaryDirectory(prefix="newsdom-ingestion-benchmark-") as root: temporary_directory = Path(root) - probe = asyncio.create_task(_event_loop_probe(stop_event, observed_delays)) + event_loop_probe = asyncio.create_task( + _event_loop_probe(stop_event, observed_delays) + ) + rss_probe = asyncio.create_task(_rss_probe(stop_event, observed_rss_bytes)) try: samples = await asyncio.gather( *( @@ -199,13 +262,18 @@ async def run_cohort( ) ) finally: + observed_rss_bytes.append(current_rss_bytes()) stop_event.set() - await probe - - maximum_delay = max(observed_delays, default=0.0) - for sample in samples: - sample["event_loop_max_delay_seconds"] = maximum_delay - return samples + await asyncio.gather(event_loop_probe, rss_probe) + + return CohortObservation( + samples=tuple(samples), + wall_seconds=time.perf_counter() - wall_started, + cpu_seconds=time.process_time() - cpu_started, + rss_baseline_bytes=rss_baseline_bytes, + peak_rss_bytes=max(observed_rss_bytes, default=0), + event_loop_delay_samples_seconds=tuple(observed_delays), + ) def _environment_record() -> dict[str, Any]: @@ -218,36 +286,48 @@ def _environment_record() -> dict[str, Any]: "platform": platform.platform(), "machine": platform.machine(), "processor_count": os.cpu_count(), + "rss_provider": "linux_proc_status_vmrss", } -def _metrics( - samples: Sequence[dict[str, int | float]], - *, - cohort_wall_seconds: Sequence[float], - cpu_seconds: Sequence[float], - peak_rss_bytes: Sequence[int], +def aggregate_metrics( + cohorts: Sequence[CohortObservation], ) -> dict[str, int | float]: - """Aggregate raw samples without discarding the samples themselves.""" + """Aggregate request and cohort evidence without lifetime RSS contamination.""" + samples = [sample for cohort in cohorts for sample in cohort.samples] + if not samples: + raise ValueError("At least one request sample is required") durations = [float(sample["duration_seconds"]) for sample in samples] total_bytes = sum(int(sample["bytes_copied"]) for sample in samples) - total_wall = sum(cohort_wall_seconds) + total_wall = sum(cohort.wall_seconds for cohort in cohorts) + loop_delays = [ + delay + for cohort in cohorts + for delay in cohort.event_loop_delay_samples_seconds + ] return { "p50_latency_seconds": nearest_rank_percentile(durations, 0.50), "p95_latency_seconds": nearest_rank_percentile(durations, 0.95), "throughput_bytes_per_second": total_bytes / total_wall if total_wall else 0.0, "read_calls": sum(int(sample["read_calls"]) for sample in samples), - "cpu_seconds": sum(cpu_seconds), - "peak_rss_bytes": max(peak_rss_bytes, default=0), - "temporary_disk_bytes": total_bytes, - "event_loop_max_delay_seconds": max( + "cpu_seconds": sum(cohort.cpu_seconds for cohort in cohorts), + "peak_rss_bytes": max( + (cohort.peak_rss_bytes for cohort in cohorts), + default=0, + ), + "peak_rss_delta_bytes": max( ( - float(sample.get("event_loop_max_delay_seconds", 0.0)) - for sample in samples + max(0, cohort.peak_rss_bytes - cohort.rss_baseline_bytes) + for cohort in cohorts ), - default=0.0, + default=0, + ), + "temporary_disk_bytes": total_bytes, + "event_loop_p95_delay_seconds": ( + nearest_rank_percentile(loop_delays, 0.95) if loop_delays else 0.0 ), + "event_loop_max_delay_seconds": max(loop_delays, default=0.0), } @@ -276,22 +356,19 @@ async def run_matrix( fixture_bytes = int(fixture_record["size_bytes"]) for candidate in candidates: chunk_bytes = resolve_chunk_bytes(candidate, fixture_bytes) - all_samples: list[dict[str, int | float]] = [] - cohort_wall_seconds: list[float] = [] - cpu_seconds: list[float] = [] - peak_rss_bytes: list[int] = [] - for _ in range(repetitions): - wall_started = time.perf_counter() - cpu_started = time.process_time() - samples = await cohort_runner( + cohorts = [ + await cohort_runner( fixture_path, chunk_bytes=chunk_bytes, concurrency=concurrency, ) - cpu_seconds.append(time.process_time() - cpu_started) - cohort_wall_seconds.append(time.perf_counter() - wall_started) - peak_rss_bytes.append(_peak_rss_bytes()) - all_samples.extend(samples) + for _ in range(repetitions) + ] + all_samples = [ + sample + for cohort in cohorts + for sample in cohort.samples + ] cases.append( { "fixture_name": fixture_path.name, @@ -299,18 +376,16 @@ async def run_matrix( "chunk_bytes": chunk_bytes, "concurrency": concurrency, "repetitions": repetitions, - "metrics": _metrics( - all_samples, - cohort_wall_seconds=cohort_wall_seconds, - cpu_seconds=cpu_seconds, - peak_rss_bytes=peak_rss_bytes, - ), + "metrics": aggregate_metrics(cohorts), + "cohort_observations": [ + cohort.as_record() for cohort in cohorts + ], "samples": all_samples, } ) report: dict[str, Any] = { - "schema_version": "1.0.0", + "schema_version": "1.1.0", "schema_path": _RESULT_SCHEMA_PATH, "generated_at": datetime.now(timezone.utc).isoformat(), "benchmark_environment": _environment_record(), From 4b044b9b473756dfabf8c3fb892189217933a74f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 7 Aug 2026 08:52:05 +0900 Subject: [PATCH 54/75] docs(benchmark): version case-local memory evidence schema --- .../upload-ingestion-result.schema.json | 71 +++++++++++++++++-- 1 file changed, 65 insertions(+), 6 deletions(-) diff --git a/docs/benchmarks/upload-ingestion-result.schema.json b/docs/benchmarks/upload-ingestion-result.schema.json index 951e2709..438d9681 100644 --- a/docs/benchmarks/upload-ingestion-result.schema.json +++ b/docs/benchmarks/upload-ingestion-result.schema.json @@ -15,7 +15,7 @@ ], "properties": { "schema_version": { - "const": "1.0.0" + "const": "1.1.0" }, "schema_path": { "const": "docs/benchmarks/upload-ingestion-result.schema.json" @@ -52,7 +52,8 @@ "executable", "platform", "machine", - "processor_count" + "processor_count", + "rss_provider" ], "properties": { "python_version": { @@ -77,6 +78,9 @@ "processor_count": { "type": ["integer", "null"], "minimum": 1 + }, + "rss_provider": { + "const": "linux_proc_status_vmrss" } } }, @@ -110,6 +114,7 @@ "concurrency", "repetitions", "metrics", + "cohort_observations", "samples" ], "properties": { @@ -143,7 +148,9 @@ "read_calls", "cpu_seconds", "peak_rss_bytes", + "peak_rss_delta_bytes", "temporary_disk_bytes", + "event_loop_p95_delay_seconds", "event_loop_max_delay_seconds" ], "properties": { @@ -171,16 +178,31 @@ "type": "integer", "minimum": 0 }, + "peak_rss_delta_bytes": { + "type": "integer", + "minimum": 0 + }, "temporary_disk_bytes": { "type": "integer", "minimum": 1 }, + "event_loop_p95_delay_seconds": { + "type": "number", + "minimum": 0 + }, "event_loop_max_delay_seconds": { "type": "number", "minimum": 0 } } }, + "cohort_observations": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/cohort_observation" + } + }, "samples": { "type": "array", "minItems": 1, @@ -190,6 +212,47 @@ } } }, + "cohort_observation": { + "type": "object", + "additionalProperties": false, + "required": [ + "wall_seconds", + "cpu_seconds", + "rss_baseline_bytes", + "peak_rss_bytes", + "peak_rss_delta_bytes", + "event_loop_delay_samples_seconds" + ], + "properties": { + "wall_seconds": { + "type": "number", + "minimum": 0 + }, + "cpu_seconds": { + "type": "number", + "minimum": 0 + }, + "rss_baseline_bytes": { + "type": "integer", + "minimum": 0 + }, + "peak_rss_bytes": { + "type": "integer", + "minimum": 0 + }, + "peak_rss_delta_bytes": { + "type": "integer", + "minimum": 0 + }, + "event_loop_delay_samples_seconds": { + "type": "array", + "items": { + "type": "number", + "minimum": 0 + } + } + } + }, "sample_record": { "type": "object", "additionalProperties": false, @@ -207,10 +270,6 @@ "type": "integer", "minimum": 1, "maximum": 20971520 - }, - "event_loop_max_delay_seconds": { - "type": "number", - "minimum": 0 } } } From bd7ef15f00301f9e5525fcf8710ab490f204220a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 7 Aug 2026 08:53:26 +0900 Subject: [PATCH 55/75] docs(benchmark): correct RSS and event-loop measurement contract --- .../upload-ingestion-benchmark-methodology.md | 83 +++++++++++-------- 1 file changed, 49 insertions(+), 34 deletions(-) diff --git a/docs/doctoring/upload-ingestion-benchmark-methodology.md b/docs/doctoring/upload-ingestion-benchmark-methodology.md index 2a9037ef..7393490a 100644 --- a/docs/doctoring/upload-ingestion-benchmark-methodology.md +++ b/docs/doctoring/upload-ingestion-benchmark-methodology.md @@ -50,10 +50,10 @@ uv run python tools/benchmark_upload_ingestion.py \ The default command runs the complete candidate and concurrency matrix. Filters are available for diagnosis, but a filtered run is not sufficient for a default -selection. Execute the benchmark on an otherwise idle, dedicated environment -and record the container or host image, CPU model, memory, storage class, -filesystem, Python and dependency lock, commit SHA, worker count, and whether -the files were cold or warm in the operating-system cache. +selection. Execute the benchmark on an otherwise idle, dedicated Linux +environment and record the container or host image, CPU model, memory, storage +class, filesystem, Python and dependency lock, commit SHA, worker count, and +whether the files were cold or warm in the operating-system cache. ## Measurement implementation @@ -61,28 +61,42 @@ the files were cold or warm in the operating-system cache. `UploadFile` and copies each request to an independent temporary file. It does not use an in-memory bytes object as the source of accepted evidence. -The harness records unaggregated samples and derives: +The harness retains both request-level and cohort-level raw observations and +derives: - p50 and p95 request latency by deterministic nearest-rank percentile; - aggregate copied bytes divided by cohort wall time; - total `UploadFile.read(size)` calls; - process CPU time; -- process peak resident set size when the platform exposes `getrusage`; +- each cohort's current-RSS baseline, sampled peak RSS, and peak-minus-baseline + delta; - temporary bytes written; -- maximum observed event-loop scheduling delay. - -Python defines `time.perf_counter()` as a high-resolution clock suitable for -short durations and `time.process_time()` as process CPU time excluding sleep. -Python's `resource.getrusage()` exposes `ru_maxrss`, but the unit is platform -dependent. The harness normalizes Linux values from KiB and Darwin values from -bytes. Windows reports zero for peak RSS in this version and cannot serve as the -accepted memory-evidence environment without an additional reviewed provider. - -Peak RSS is a process high-water mark rather than an isolated per-case delta. -Accordingly, execute candidates in separate fresh processes when making the -final memory decision, or treat later in-process values as conservative upper -bounds. Do not subtract two high-water marks and call the result allocated -memory. +- raw event-loop scheduling-delay samples, their nearest-rank p95, and maximum. + +Python defines `time.perf_counter()` as a high-resolution monotonic clock suited +to duration measurement and `time.process_time()` as process CPU time excluding +sleep. Event-loop responsiveness is reported as a real nearest-rank p95 over the +retained probe samples, not as a maximum mislabeled as a percentile. + +On Linux, the harness samples `VmRSS` from `/proc/self/status` during each +cohort. The Linux kernel distinguishes `VmRSS`, the current resident set, from +`VmHWM`, the process-lifetime resident-set high-water mark. A lifetime high-water +metric would contaminate later matrix cases with memory used by earlier cases; +therefore `resource.getrusage(...).ru_maxrss` is not accepted as case-local +memory evidence. Every report preserves the cohort baseline and sampled peak so +the absolute process budget and incremental case pressure can both be audited. + +If `/proc/self/status` is unavailable or malformed, the provider returns zero +and the environment record still identifies `linux_proc_status_vmrss`. Such a +run does not satisfy the final memory-evidence gate. Other operating systems +require a separately reviewed current-RSS provider before their memory numbers +can support a production default. + +RSS sampling is observational rather than an allocation profiler. Candidate +selection therefore requires repeated clean-process runs and should be +corroborated with operating-system or container memory telemetry. Do not treat +`peak_rss_delta_bytes` as exact ownership of memory that may be shared, +allocator-retained, or concurrently used by another thread. ## Raw evidence schema @@ -92,11 +106,11 @@ The checked-in JSON Schema is: docs/benchmarks/upload-ingestion-result.schema.json ``` -Accepted evidence must validate against schema version `1.0.0`. The report keeps -raw samples alongside aggregates, rejects unknown top-level and case fields, -and records the exact resolved byte size even for the adaptive candidate. -Generated reports are evidence artifacts, not source files; do not commit -private fixture paths or customer document names. +Accepted evidence must validate against schema version `1.1.0`. The report keeps +request samples and raw cohort observations alongside aggregates, rejects +unknown top-level and case fields, and records the exact resolved byte size even +for the adaptive candidate. Generated reports are evidence artifacts, not source +files; do not commit private fixture paths or customer document names. ## Interpretation and selection gates @@ -105,8 +119,10 @@ of the following across the complete matrix: 1. no upload-size, magic-byte, temporary-file cleanup, authentication, or parser admission regression; -2. no materially worse p95 latency or event-loop delay at high concurrency; -3. peak RSS and temporary storage remain inside an explicit per-process budget; +2. no materially worse request p95 latency or event-loop p95 at high + concurrency, with the maximum retained for tail diagnosis; +3. absolute sampled peak RSS and peak-minus-baseline pressure remain inside an + explicit per-process budget; 4. thread-pool read-call reduction does not merely shift cost to larger retained buffers or storage stalls; 5. results are reproduced on at least two runs from clean processes; @@ -115,9 +131,9 @@ of the following across the complete matrix: release evidence all identify the same value or rule. Compare distributions and operational thresholds rather than publishing a -single mean. A percentage improvement may be stated only when the checked-in or -attached raw report, environment manifest, analysis code, and confidence or -variability treatment support it. +single mean. A percentage improvement may be stated only when the attached raw +report, environment manifest, analysis code, and variability treatment support +it. ## Accuracy and security invariants @@ -150,12 +166,11 @@ loaded, and rerun the overload and upload-limit smoke tests. FastAPI. (2026). *Request files*. FastAPI documentation. https://fastapi.tiangolo.com/tutorial/request-files/ -Python Software Foundation. (2026). *resourceโ€”Resource usage information -(Python 3.10.20 documentation)*. -https://docs.python.org/3.10/library/resource.html +Linux Kernel Organization. (2026). *The /proc filesystem*. Linux kernel +documentation. https://www.kernel.org/doc/html/latest/filesystems/proc.html Python Software Foundation. (2026). *timeโ€”Time access and conversions (Python -3.10.20 documentation)*. https://docs.python.org/3.10/library/time.html +3.14.6 documentation)*. https://docs.python.org/3/library/time.html Starlette. (2026). *Thread pool*. Starlette documentation. https://www.starlette.io/threadpool/ From 8552a689a31db1f6c384d838da3095bbc15d1e52 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 10:11:15 +0900 Subject: [PATCH 56/75] docs: make NewsDOM landing product-first and license-aware --- README.md | 273 ++++++++++++++++++------------------------------------ 1 file changed, 90 insertions(+), 183 deletions(-) diff --git a/README.md b/README.md index 461fcda8..a7c68b9b 100644 --- a/README.md +++ b/README.md @@ -1,49 +1,42 @@ # NewsDOM API -[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/Seongho-Bae/newsdom-api/badge)](https://securityscorecards.dev/viewer/?uri=github.com/Seongho-Bae/newsdom-api) +[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/ContextualWisdomLab/newsdom-api/badge)](https://securityscorecards.dev/viewer/?uri=github.com/ContextualWisdomLab/newsdom-api) +[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/ContextualWisdomLab/newsdom-api) -NewsDOM API is a language-agnostic PDF-to-DOM sidecar. It converts any PDF -into a canonical JSON document tree โ€” pages, sections, headings, body -blocks, images, captions, and bounding boxes โ€” using MinerU. It runs as a -standalone FastAPI service and is designed to be embedded as a git -submodule / sidecar in a larger system. +**Turn PDFs into a stable, structured document tree that downstream systems can inspect instead of reverse-engineering parser output.** -## Features +NewsDOM API owns the HTTP, safety, normalization, and evidence boundary for PDF-to-DOM processing. It accepts a PDF, applies bounded admission and fail-closed request controls, and returns canonical JSON for pages, sections, headings, body blocks, images, captions, bounding boxes, and quality metadata. -- Primary engine: `MinerU` pipeline backend -- Service wrapper: FastAPI -- Output: canonical JSON with pages, sections, section headings, body - blocks, images, captions, bounding boxes, and quality metadata -- Language-selectable: `language` defaults to MinerU's multilingual `ch` model; - official language families and compatibility aliases such as `japan` remain - available -- Parsing `mode` defaults to `auto` so born-digital text PDFs skip forced OCR -- Fail-closed bearer authentication on `/parse`; separate unauthenticated `/health` liveness and `/ready` traffic-readiness probes -- Bounded, non-waiting parser admission with a conservative one-parse default per process and an explicit `429 Too Many Requests` overload contract +The service is intentionally usable as a standalone FastAPI sidecar. A host application does not need to adopt NewsDOM's internal parser process, storage layout, or deployment model; it consumes the published HTTP/schema contract. -## Quickstart +> **Commercial parser status:** the current source still contains a legacy MinerU adapter, but MinerU 3.x uses the MinerU Open Source Licenseโ€”Apache-2.0 plus additional commercial conditions. That runtime is **not an approved ContextualWisdomLab commercial dependency**. Do not treat MinerU installation, a customer-supplied MinerU binary, or `Dockerfile.nvidia` as an approved commercial deployment path. Issue [#671](https://github.com/ContextualWisdomLab/newsdom-api/issues/671) owns replacement with an unrestricted commercial-use parser boundary. NewsDOM's own source remains MIT-licensed. -### Install +## What NewsDOM owns -Install `uv` first if it is not already available in your `PATH`, then sync the -repository-managed virtual environment: +- A stable `POST /parse` contract for PDF input and normalized document-tree output. +- Language and parse-mode request validation, including `mode=auto`, `ocr`, and `txt` compatibility semantics while the legacy adapter remains present. +- Fail-closed bearer authentication for `/parse` in the production profile. +- Separate unauthenticated `/health` liveness and `/ready` traffic-readiness semantics. +- Bounded, non-waiting parser admission with `NEWSDOM_MAX_CONCURRENT_PARSES` and an explicit `429 Too Many Requests` overload response. +- Request-scoped temporary workspace handling, sanitized parser failures, schema validation, and synthetic fixture provenance. -```bash -uv sync --frozen --all-extras -``` +NewsDOM does **not** make a third-party parser's license, model weights, OCR accuracy, or runtime availability part of its own MIT grant. It also does not turn `/health` into proof that parsing is ready. + +## Quick start -To enable real parsing with MinerU, install the MinerU CLI separately in the -same `.venv` that `uv sync` created: +### Install the repository-managed environment + +NewsDOM source metadata currently requires Python `>=3.10,<3.14`. ```bash -uv pip install --python .venv/bin/python "mineru[pipeline]==3.4.4" +uv sync --frozen --all-extras ``` -On Windows, replace `.venv/bin/python` with `.venv\Scripts\python.exe`. +`uv` creates the repository environment under `.venv`. On Windows the interpreter path is `.venv\Scripts\python.exe`; on macOS/Linux it is `.venv/bin/python`. -### Run +This command installs NewsDOM and its declared project dependencies only. It does **not** install the commercially restricted legacy MinerU runtime. -Configure the fail-closed production contract before starting the service: +### Run the API shell ```bash export NEWSDOM_AUTH_MODE=required @@ -53,169 +46,96 @@ export NEWSDOM_MAX_CONCURRENT_PARSES=1 uv run uvicorn --app-dir src newsdom_api.main:app --reload ``` -### Docker +Check the two health levels independently: ```bash -docker build -t newsdom-api . -docker run -p 8000:8000 \ - -e NEWSDOM_AUTH_MODE=required \ - -e NEWSDOM_RUNTIME_PROFILE=production \ - -e NEWSDOM_API_TOKEN="$NEWSDOM_API_TOKEN" \ - -e NEWSDOM_MAX_CONCURRENT_PARSES=1 \ - newsdom-api +curl -sS http://127.0.0.1:8000/health +curl -sS http://127.0.0.1:8000/ready ``` -The default image exposes the REST API on port `8000` as a multi-arch service -image. It is suitable for `linux/amd64` and `linux/arm64`, including Apple -Silicon hosts running the API service inside Docker. +`/health` answers for process liveness. `/ready` is the routing gate and must remain unavailable when the configured parser runtime or required authentication state cannot accept traffic. -The default image ships the API service only and does not bundle the MinerU runtime. -`/parse` requires a compatible MinerU runtime to be available inside the container image or exposed through `NEWSDOM_MINERU_BIN`. +### Parse contract -> `GET /health` reports process liveness only. `GET /ready` reports traffic -> readiness and remains unavailable until both fail-closed authentication and -> the MinerU runtime are configured. Orchestrators must route traffic by -> `/ready`, not by `/health`. - -#### docker compose - -A production-oriented `docker-compose.yml` is provided for standalone/sidecar -use. It requires `NEWSDOM_API_TOKEN` before startup, sets the explicit -one-parse process budget, and its healthcheck targets `/ready`: +Once an **organization-approved parser backend** is configured, the public request shape remains: ```bash -docker compose up --build -``` - -Export `NEWSDOM_API_TOKEN` from a secret store before running Compose. Mount or -bundle MinerU and set `NEWSDOM_MINERU_BIN` when using the API-only image. The -service remains live but not ready until both dependencies are valid. - -#### Building a MinerU-bundled image - -To ship an image where `/parse` works behind a green `/health`, bundle a MinerU -runtime. Either build the NVIDIA variant below, or extend the default image and -install MinerU into the same virtualenv, for example: - -```dockerfile -FROM newsdom-api:latest -USER root -RUN uv pip install --python /app/.venv/bin/python "mineru[pipeline]==3.4.4" -USER newsdom +curl -F "file=@sample.pdf" \ + -H "Authorization: Bearer $NEWSDOM_API_TOKEN" \ + http://127.0.0.1:8000/parse ``` -Alternatively mount a MinerU executable and point `NEWSDOM_MINERU_BIN` at it, so -`/parse` no longer returns `503` behind a healthy `/health`. - -For heavier parsing deployments, build the optional NVIDIA-oriented variant: +Optional compatibility fields are `language` and `mode`: ```bash -docker build -f Dockerfile.nvidia -t newsdom-api:nvidia . -docker run --gpus all -p 8000:8000 newsdom-api:nvidia +curl -F "file=@sample.pdf" -F "language=ch" -F "mode=auto" \ + -H "Authorization: Bearer $NEWSDOM_API_TOKEN" \ + http://127.0.0.1:8000/parse ``` -`Dockerfile.nvidia` is intended for Linux/NVIDIA environments and is -`linux/amd64`-only. Apple Silicon can run the lean API image, but Docker -Desktop does not expose Apple GPU acceleration to Linux containers, so real -GPU-accelerated parsing should stay on a native Apple Silicon path instead of -the containerized runtime. - -The NVIDIA variant is `linux/amd64`-only and is meant for hosts that can -provide the CUDA user-space/runtime stack required by MinerU. - -### Parse a PDF +An explicit OCR-mode request uses the same authenticated boundary: ```bash -curl -F "file=@sample.pdf" \ +curl -F "file=@sample.pdf" -F "language=korean" -F "mode=ocr" \ -H "Authorization: Bearer $NEWSDOM_API_TOKEN" \ http://127.0.0.1:8000/parse ``` -`/parse` accepts `multipart/form-data` with a required `file` part -(`application/pdf`) and two optional form fields: +The current source's legacy adapter may return `503 Service Unavailable` when no parser runtime is available. That is preferable to silently treating an unapproved runtime as production-ready. -| Field | Default | Values | Maps to | -| ----- | ------- | ------ | ------- | -| `language` | `ch` | MinerU 3.4.4 public family or alias (`ch`, `en`, `japan`, `korean`, `arabic`, `east_slavic`, `cyrillic`, `devanagari`, โ€ฆ) | MinerU `-l` | -| `mode` | `auto` | `auto`, `ocr`, `txt` | MinerU `-m` | +## Capacity and failure behavior -`mode=auto` lets born-digital (text-layer) PDFs skip forced OCR; `ocr` forces -optical recognition and `txt` extracts only the embedded text layer. Invalid -values return `422`. The previous Japanese-newspaper behavior is still available -explicitly: +`NEWSDOM_MAX_CONCURRENT_PARSES` is an integer from `1` through `128` and defaults to `1` per process. Authentication runs before admission. When all parser leases are occupied, the service returns `429 Too Many Requests` with `Retry-After: 1` and `Cache-Control: no-store` before multipart body processing, temporary-file allocation, PDF validation, or parser execution. -```bash -curl -F "file=@sample.pdf" -F "language=japan" -F "mode=ocr" \ - -H "Authorization: Bearer $NEWSDOM_API_TOKEN" \ - http://127.0.0.1:8000/parse -``` +This is a last-resort per-process availability boundary, not a replacement for tenant-aware gateway limits or cluster resource quotas. Start with the conservative default, measure representative workloads, and scale replicas before raising per-process concurrency. -The accepted language contract follows the official -[MinerU 3.4.4 CLI implementation](https://github.com/opendatalab/MinerU/blob/mineru-3.4.4-released/mineru/utils/ocr_language.py). -MinerU canonicalizes `en`, `japan`, `chinese_cht`, and `latin` to `ch`; this -sidecar performs the same normalization before launching the subprocess. +Expected malformed-document/parser failures are translated into bounded client-facing responses; unexpected server faults remain observable through the server-error path rather than being mislabeled as caller input. -#### Authentication, readiness, and parser capacity +## Containers -Parser authentication is required by default. This replaces the previous -default-open behavior. Configure the explicit production contract and supply the -secret from the deployment secret store: +The default container is the supported repository image surface: ```bash -export NEWSDOM_AUTH_MODE=required -export NEWSDOM_RUNTIME_PROFILE=production -export NEWSDOM_API_TOKEN=$(openssl rand -hex 32) -export NEWSDOM_MAX_CONCURRENT_PARSES=1 -uv run uvicorn --app-dir src newsdom_api.main:app -curl -F "file=@sample.pdf" -H "Authorization: Bearer $NEWSDOM_API_TOKEN" \ - http://127.0.0.1:8000/parse +docker build -t newsdom-api . +docker run -p 8000:8000 \ + -e NEWSDOM_AUTH_MODE=required \ + -e NEWSDOM_RUNTIME_PROFILE=production \ + -e NEWSDOM_API_TOKEN="$NEWSDOM_API_TOKEN" \ + -e NEWSDOM_MAX_CONCURRENT_PARSES=1 \ + newsdom-api ``` -Missing or invalid caller credentials receive a fixed `401`; a missing required -server token returns a fixed `503` before the upload body is processed. `GET -/health` remains unauthenticated liveness. `GET /ready` succeeds only when the -authentication configuration and MinerU runtime can accept traffic. - -`NEWSDOM_MAX_CONCURRENT_PARSES` is an immutable integer from `1` through `128` -and defaults to `1`. It controls active MinerU work **per process**. Authentication -runs first. An authenticated request then obtains one non-waiting lease before the -multipart body is parsed or copied. When every lease is in use, the service returns -`429 Too Many Requests` with `Retry-After: 1` and `Cache-Control: no-store` before -the multipart body, temporary-file allocation, PDF validation, or MinerU execution. -The service deliberately does not create an unbounded in-process queue. - -Total nominal capacity is the process budget multiplied by the number of serving -processes and each replica. Start with one lease, measure peak RSS/VRAM and parser -latency on representative PDFs, and scale replicas before increasing per-process -capacity. Clients should apply bounded, jittered retries and honor `Retry-After: 1`. -The limit is availability protection rather than a substitute for gateway-level -per-tenant rate limits or cluster resource quotas. - -An isolated local development-only bypass is available only with the explicit -pair below: +The default image **ships the API service only** and **does not bundle the MinerU runtime**. It supports the repository's multi-architecture API shell, including Apple Silicon hosts running Linux containers, but `/ready` must stay fail-closed until an approved parser backend exists. -```bash -NEWSDOM_AUTH_MODE=disabled \ -NEWSDOM_RUNTIME_PROFILE=development \ - uv run uvicorn --app-dir src newsdom_api.main:app --reload +`Dockerfile.nvidia` is retained in current source history and tests as the existing NVIDIA/MinerU distribution path, but it is **commercially blocked under organization policy** and must not be built, published, recommended, or treated as a supported release profile until #671 replaces the restricted parser stack. The same restriction applies even on an NVIDIA host; moving the component into another image or process does not cure its license terms. + +## Architecture and integration boundary + +```text +client + -> FastAPI request/auth boundary + -> non-waiting parser admission + -> bounded temporary PDF workspace + -> parser adapter port + -> NewsDOM normalization + schema validation + -> canonical JSON response ``` -Do not use the development-only bypass as a production rollback. Restore a -working secret or the previous release instead. +The parser adapter is an implementation dependency. The stable product responsibility is the request, safety, evidence, and normalized-output contract around it. The replacement work in #671 should preserve that boundary rather than make downstream callers depend on another parser's internal types. + +For deeper design and current-vs-target maturity, see [ARCHITECTURE.md](ARCHITECTURE.md), the [ADR index](docs/adr/README.md), and the canonical documentation under [`docs/`](docs/). -Each request is written to a request-scoped temporary directory before MinerU -runs, and those temporary files are removed after the response completes. -Sanitized parse failures return `503 Service Unavailable` when the -runtime cannot be executed and `502 Bad Gateway` when MinerU -finishes without the required output artifacts. +## Verification -### Run tests +Run the repository suite: ```bash uv run pytest ``` -### Fuzzing smoke +The repository quality gate requires 100% configured source coverage and docstring audit coverage. Security, dependency, image, fuzzing, and package checks run independently in CI. + +A local fuzzing smoke remains available against committed synthetic fixtures: ```bash uv run python fuzzers/dom_builder_fuzzer.py --smoke tests/fixtures/mineru_sample.json @@ -223,42 +143,29 @@ uv run python fuzzers/schema_response_fuzzer.py --smoke fuzzers/corpus/schema_re uv run python fuzzers/equivalence_metrics_fuzzer.py --smoke fuzzers/corpus/equivalence_metrics_fuzzer/structural_metrics.json ``` -Every `fuzzers/*_fuzzer.py` target is coverage-guided under Atheris and is -picked up automatically by the ClusterFuzzLite workflow, which runs a bounded -budget on each pull request. Targets cover the untrusted-input boundaries: the -MinerU DOM normalizer (`build_dom`), the `ParseResponse` schema validator, and -the equivalence metrics normalizer. See `docs/papers/` for background. - -The repository also enforces a `quality-gate` workflow with 100% source -coverage and docstring audit coverage. - -## Fixtures and provenance +The historical `mineru_sample.json` fixture name records parser provenance; it is not an instruction to install the MinerU runtime. -This repository ships only synthetic test fixtures and derived structural -baselines. For fixture provenance and regeneration notes, see -`tests/fixtures/README.md`. +## Documentation -## Development +- [Public manual](manual/index.md) โ€” product and integration entry point. +- [Architecture](ARCHITECTURE.md) โ€” system ownership and current/target boundaries. +- [Security](SECURITY.md) โ€” vulnerability reporting and security posture. +- [Contributing](CONTRIBUTING.md) โ€” development and validation workflow. +- [Fixtures and provenance](tests/fixtures/README.md) โ€” synthetic fixture source and regeneration rules. +- [Git flow](docs/workflow/git-flow.md) โ€” repository branch workflow. +- [Changelog](CHANGELOG.md) โ€” source change history; not release evidence by itself. +- [Commercial parser replacement #671](https://github.com/ContextualWisdomLab/newsdom-api/issues/671) โ€” current parser-license blocker and acceptance contract. -Development setup, fixture handling rules, and local-only baseline -maintenance are documented in `CONTRIBUTING.md`. +GitHub Pages source lives under `manual/`. A source commit is not proof that the public site deployed; publication requires the Pages workflow to finish and the live HTTPS content to be re-read. -Mechanical branch updates and merges are attributed to `github-actions[bot]`. -Scratch PoC files are not committed. Failed GitHub Checks are not reviewed as URL lists. -OpenCode Review, Strix Security Scan, and PR Review Merge Scheduler are -provided by the organization-level required workflows in -`ContextualWisdomLab/.github`, not copied into this repository. +## Status -Security reporting guidance is documented in `SECURITY.md`. -Version tags trigger a GitHub-native release workflow that builds -distribution artifacts, checksums, and provenance attestations. +`pyproject.toml` currently records NewsDOM source version `0.2.0`. Source metadata, an open PR, or a passing check is not by itself a published-release or commercial-readiness claim. Consult [GitHub Releases](https://github.com/ContextualWisdomLab/newsdom-api/releases) for immutable published release evidence. -Project history is tracked in `CHANGELOG.md`. +The current parser-license blocker means the repository must not be represented as commercially complete even when NewsDOM-owned tests are green. -Repository branch workflow is documented in `docs/workflow/git-flow.md`. +## License -## Repository layout +NewsDOM API original source and documentation are licensed under the [MIT License](LICENSE). Third-party packages, parser runtimes, model weights, container bases, fonts, datasets, and other external assets retain their own licenses and distribution terms. -- `src/newsdom_api/`: API, MinerU wrapper, admission control, DOM builder, synthetic fixture generator -- `tests/`: unit tests and committed synthetic fixtures -- `tools/`: local maintenance utilities +The MIT grant does not relicense the legacy MinerU runtime. Under current ContextualWisdomLab policy, that runtime is outside the approved commercial inbound baseline and remains blocked by #671. From 0ae9f94ec06453c1d7129622db58e057408eeaa1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 10:11:40 +0900 Subject: [PATCH 57/75] docs: remove restricted parser install path --- manual/installation.md | 85 ++++++++++++++++-------------------------- 1 file changed, 32 insertions(+), 53 deletions(-) diff --git a/manual/installation.md b/manual/installation.md index 754327de..e84526c4 100644 --- a/manual/installation.md +++ b/manual/installation.md @@ -1,87 +1,66 @@ # ์‹œ์Šคํ…œ ํ™˜๊ฒฝ ๋ฐ ์„ค์น˜ ๊ฐ€์ด๋“œ -์ด ๋ฌธ์„œ์—์„œ๋Š” NewsDOM API๋ฅผ ๋กœ์ปฌ ํ™˜๊ฒฝ์— ์„ค์น˜ํ•˜๋Š” ๋ฐฉ๋ฒ•์„ ์•ˆ๋‚ดํ•ฉ๋‹ˆ๋‹ค. +์ด ๋ฌธ์„œ๋Š” **NewsDOM API ์ž์ฒด**๋ฅผ ๊ฐœ๋ฐœยท๊ฒ€์ฆํ•˜๊ธฐ ์œ„ํ•œ ์„ค์น˜ ์ ˆ์ฐจ๋ฅผ ์„ค๋ช…ํ•ฉ๋‹ˆ๋‹ค. ํ˜„์žฌ ์†Œ์Šค์˜ legacy MinerU adapter๋Š” ๋ณ„๋„ parser runtime์„ ํ˜ธ์ถœํ•˜์ง€๋งŒ, MinerU 3.x์˜ ์ถ”๊ฐ€ ์ƒ์—… ๋ผ์ด์„ ์Šค ์กฐ๊ฑด์€ ContextualWisdomLab์˜ unrestricted commercial-inbound ์ •์ฑ…๊ณผ ํ˜ธํ™˜๋˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ๋”ฐ๋ผ์„œ ์ด ๋ฌธ์„œ๋Š” MinerU ์„ค์น˜๋ฅผ ์ง€์› ๊ฒฝ๋กœ๋กœ ์•ˆ๋‚ดํ•˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ๋Œ€์ฒด parser boundary๋Š” [#671](https://github.com/ContextualWisdomLab/newsdom-api/issues/671)์—์„œ ์ถ”์ ํ•ฉ๋‹ˆ๋‹ค. -## ๐Ÿ› ๏ธ ์‹œ์Šคํ…œ ์š”๊ตฌ์‚ฌํ•ญ +## ์‹œ์Šคํ…œ ์š”๊ตฌ์‚ฌํ•ญ - **Python**: Required: `>=3.10, <3.14` -- **์šด์˜์ฒด์ œ**: Linux ๋˜๋Š” macOS ๊ถŒ์žฅ (์œˆ๋„์šฐ์˜ ๊ฒฝ์šฐ WSL2 ์‚ฌ์šฉ ๊ถŒ์žฅ) -- **ํ•˜๋“œ์›จ์–ด (GPU)**: `MinerU` ๋”ฅ๋Ÿฌ๋‹ ๊ธฐ๋ฐ˜ ํŒŒ์ดํ”„๋ผ์ธ์„ ๊ตฌ๋™ํ•˜๊ธฐ ์œ„ํ•ด์„œ๋Š” ์ตœ์†Œ **8GB ์ด์ƒ์˜ RAM**์ด ํ•„์š”ํ•˜๋ฉฐ, ์‹ค์‹œ๊ฐ„ ์ฒ˜๋ฆฌ๋ฅผ ์œ„ํ•ด **NVIDIA GPU(CUDA 11.x/12.x ํ˜ธํ™˜)** ๋ฐ `PyTorch` ํ™˜๊ฒฝ์ด ๊ถŒ์žฅ๋ฉ๋‹ˆ๋‹ค. -- **์˜์กด์„ฑ (Python)**: - - `fastapi>=0.115,<1.0`, `uvicorn>=0.30,<1.0`, `pydantic>=2.9,<3.0` - - `python-multipart`, `reportlab`, `Pillow`, `pypdf` ๋“ฑ +- **์šด์˜์ฒด์ œ**: Linux, macOS, Windows์˜ ์ง€์› Python ํ™˜๊ฒฝ +- **ํŒจํ‚ค์ง€ ๊ด€๋ฆฌ์ž**: `uv` +- **NewsDOM ์˜์กด์„ฑ**: `pyproject.toml` / `uv.lock`์ด ๊ถŒ์œ„ ์žˆ๋Š” ์„ค์น˜ ๊ณ„์•ฝ ---- +GPU, OCR model, parser model weight๋Š” NewsDOM ์ž์ฒด ์„ค์น˜ ์š”๊ตฌ์‚ฌํ•ญ์œผ๋กœ ๊ฐ„์ฃผํ•˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ์Šน์ธ๋œ parser backend๊ฐ€ ๋„์ž…๋˜๋ฉด ํ•ด๋‹น backend์˜ ํ”Œ๋žซํผยท๋ฆฌ์†Œ์Šคยท๋ผ์ด์„ ์Šค ์š”๊ตฌ์‚ฌํ•ญ์„ ๋ณ„๋„ profile๋กœ ๋ฌธ์„œํ™”ํ•ด์•ผ ํ•ฉ๋‹ˆ๋‹ค. -## 1. ๊ธฐ๋ณธ ํ…Œ์ŠคํŠธ ๋ฐ ๊ฐœ๋ฐœ ๋ชจ๋“œ ์„ค์น˜ +## 1. ์ €์žฅ์†Œ ํ™˜๊ฒฝ ์„ค์น˜ -๊ฐ€์žฅ ๊ฐ„๋‹จํ•œ ํ˜•ํƒœ๋กœ ํŒŒ์ด์ฌ ๊ฐ€์ƒํ™˜๊ฒฝ(Virtual Environment)์„ ์ƒ์„ฑํ•˜๊ณ  ํŒจํ‚ค์ง€๋ฅผ ์„ค์น˜ํ•ฉ๋‹ˆ๋‹ค. ์ด ๋ชจ๋“œ์—์„œ๋Š” ์‹ค์ œ `MinerU` ๋ชจ๋ธ์ด ๋กœ๋“œ๋˜์ง€ ์•Š์œผ๋ฉฐ, `pytest`๋‚˜ ํ•ฉ์„ฑ ํ”ฝ์Šค์ฒ˜(Synthetic Fixtures) ๊ธฐ๋ฐ˜ ํ…Œ์ŠคํŠธ ์šฉ๋„๋กœ ์ ํ•ฉํ•ฉ๋‹ˆ๋‹ค. ์˜ˆ์‹œ ๋ช…๋ น์€ `python3.10`์„ ์‚ฌ์šฉํ•˜์ง€๋งŒ, ์ง€์› ๋ฒ”์œ„ ์•ˆ์˜ ๋‹ค๋ฅธ ์ธํ„ฐํ”„๋ฆฌํ„ฐ๋„ ๋™์ผํ•˜๊ฒŒ ์‚ฌ์šฉํ•  ์ˆ˜ ์žˆ์Šต๋‹ˆ๋‹ค. +์˜ˆ์‹œ๋Š” `python3.10`์„ ์‚ฌ์šฉํ•˜์ง€๋งŒ, ์ง€์› ๋ฒ”์œ„ ์•ˆ์˜ ๋‹ค๋ฅธ ์ธํ„ฐํ”„๋ฆฌํ„ฐ๋„ ์‚ฌ์šฉํ•  ์ˆ˜ ์žˆ์Šต๋‹ˆ๋‹ค. `uv`๋ฅผ ์ด์šฉํ•œ canonical setup์€ ๋‹ค์Œ๊ณผ ๊ฐ™์Šต๋‹ˆ๋‹ค. ```bash -# ๊ฐ€์ƒ ํ™˜๊ฒฝ ์ƒ์„ฑ (๊ถŒ์žฅ ์˜ˆ์‹œ: python3.10) -python3.10 -m venv .venv +uv sync --frozen --all-extras +``` -# ๊ฐ€์ƒ ํ™˜๊ฒฝ ํ™œ์„ฑํ™” -# macOS / Linux -source .venv/bin/activate -# Windows (WSL ํ™˜๊ฒฝ ์ œ์™ธ) -# .venv\Scripts\activate +๋™์ผํ•œ ๊ฐ€์ƒํ™˜๊ฒฝ ์œ„์น˜๋ฅผ ์ง์ ‘ ํ™•์ธํ•ด์•ผ ํ•˜๋Š” ๊ฒฝ์šฐ macOS/Linux์—์„œ๋Š” `.venv/bin/python`, Windows์—์„œ๋Š” `.venv\Scripts\python.exe`๋ฅผ ์‚ฌ์šฉํ•ฉ๋‹ˆ๋‹ค. -# pip ์—…๊ทธ๋ ˆ์ด๋“œ -python -m pip install --upgrade pip +๊ณผ๊ฑฐ ์ˆ˜๋™ ์„ค์น˜ ์˜ˆ์‹œ์ธ `python3.10 -m venv .venv`์™€ `pip install -e ".[dev]"`๋Š” ํ˜ธํ™˜์„ฑ ์ฐธ๊ณ ์šฉ์ผ ๋ฟ canonical repository setup์ด ์•„๋‹™๋‹ˆ๋‹ค. ํ˜„์žฌ ๊ฒ€์ฆ๊ณผ CI๋Š” lockfile ๊ธฐ๋ฐ˜ `uv` ํ™˜๊ฒฝ์„ ๊ธฐ์ค€์œผ๋กœ ํ•ฉ๋‹ˆ๋‹ค. -# ์˜์กด์„ฑ ํŒจํ‚ค์ง€์™€ ํ•จ๊ป˜ ๊ฐœ๋ฐœ ๋ชจ๋“œ๋กœ ์„ค์น˜ -pip install -e ".[dev]" -``` +## 2. parser runtime ์ •์ฑ… ---- +ํ˜„์žฌ repository source์—๋Š” legacy MinerU adapter๊ฐ€ ๋‚จ์•„ ์žˆ์ง€๋งŒ ๋‹ค์Œ ๊ฒฝ๋กœ๋Š” **์ƒ์—… ๋ฐฐํฌ ์Šน์ธ ๊ฒฝ๋กœ๊ฐ€ ์•„๋‹™๋‹ˆ๋‹ค**. -## 2. MinerU ๋ฐฑ์—”๋“œ ํฌํ•จ ์‹ค์ œ ํŒŒ์‹ฑ ๋ชจ๋“œ ์„ค์น˜ +- MinerU Python package ์ง์ ‘ ์„ค์น˜ +- `NEWSDOM_MINERU_BIN`์œผ๋กœ ๊ณ ๊ฐ์ด ๊ณต๊ธ‰ํ•œ MinerU binary ์—ฐ๊ฒฐ +- `Dockerfile.nvidia`๋ฅผ ์ด์šฉํ•œ MinerU-bundled image ๋นŒ๋“œ/๋ฐฐํฌ +- MinerU๋ฅผ ๋ณ„๋„ container/process/service๋กœ ์˜ฎ๊ธด ๋’ค NewsDOM์˜ ํ•„์ˆ˜ runtime์œผ๋กœ ์‚ฌ์šฉํ•˜๋Š” ๊ตฌ์„ฑ -`MinerU` ๋ฐฑ์—”๋“œ๋ฅผ ์‚ฌ์šฉํ•˜์—ฌ ์‹ค์ œ ์Šค์บ”๋œ ์ผ๋ณธ์–ด ์‹ ๋ฌธ PDF ํŒŒ์‹ฑ ์ž‘์—…์„ ์ˆ˜ํ–‰ํ•˜๋ ค๋ฉด MinerU CLI๋ฅผ ๋ณ„๋„๋กœ ์„ค์น˜ํ•ด์•ผ ํ•ฉ๋‹ˆ๋‹ค. +์ด ์ œํ•œ์€ ๋‹จ์ˆœ attribution ๋ฌธ์ œ๊ฐ€ ์•„๋‹ˆ๋ผ upstream์˜ ์ถ”๊ฐ€ commercial threshold ์กฐ๊ฑด๊ณผ ์กฐ์ง์˜ inbound ์ •์ฑ… ์ฐจ์ด ๋•Œ๋ฌธ์ž…๋‹ˆ๋‹ค. MIT๋กœ ๋ฐฐํฌ๋˜๋Š” NewsDOM ์ž์ฒด๊ฐ€ ์ œ3์ž parser๋ฅผ ์žฌ๋ผ์ด์„ ์Šคํ•˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. -```bash -# MinerU ํŒŒ์ดํ”„๋ผ์ธ CLI ์„ค์น˜ -pip install "mineru[pipeline]==3.4.4" -``` +์Šน์ธ๋œ parser backend๊ฐ€ ์•„์ง ์—†๋Š” ํ™˜๊ฒฝ์—์„œ `/ready`๊ฐ€ ์‹คํŒจํ•˜๋Š” ๊ฒƒ์€ ์˜ฌ๋ฐ”๋ฅธ fail-closed ์ƒํƒœ์ž…๋‹ˆ๋‹ค. parser๋ฅผ ์šฐํšŒํ•˜๊ฑฐ๋‚˜ readiness๋ฅผ ๊ฐ•์ œ๋กœ green์œผ๋กœ ๋งŒ๋“ค์ง€ ๋งˆ์‹ญ์‹œ์˜ค. -์ด ๋ช…๋ น์–ด๋ฅผ ํ†ตํ•ด **`mineru[pipeline]==3.4.4`** ๋ฒ„์ „์ด ์„ค์น˜๋˜๋ฉฐ ๋”ฅ๋Ÿฌ๋‹ ๊ธฐ๋ฐ˜ ๋ชจ๋ธ์„ ์œ„ํ•œ ์ค€๋น„๊ฐ€ ์™„๋ฃŒ๋ฉ๋‹ˆ๋‹ค. ์„ค์น˜ ํ›„ ์ฒ˜์Œ API ์„œ๋ฒ„๋ฅผ ๊ตฌ๋™ํ•˜๊ณ  PDF๋ฅผ ํŒŒ์‹ฑํ•  ๋•Œ ๋ชจ๋ธ(Weight) ํŒŒ์ผ์„ ๋ฐฑ๊ทธ๋ผ์šด๋“œ์—์„œ ๋‹ค์šด๋กœ๋“œํ•  ์ˆ˜ ์žˆ์œผ๋ฏ€๋กœ, ์ฒซ ์‹คํ–‰์—๋Š” ๋‹ค์šด๋กœ๋“œ ๋Œ€๊ธฐ ์‹œ๊ฐ„์ด ๋ฐœ์ƒํ•  ์ˆ˜ ์žˆ์Šต๋‹ˆ๋‹ค. +## 3. ํ…Œ์ŠคํŠธ์™€ API shell ํ™•์ธ -### ์ปค์Šคํ…€ MinerU ์‹คํ–‰ ๊ฒฝ๋กœ (๊ณ ๊ธ‰) -๋งŒ์•ฝ `mineru` CLI ๋ฐ”์ด๋„ˆ๋ฆฌ๊ฐ€ ์‹œ์Šคํ…œ PATH์— ์žกํ˜€์žˆ์ง€ ์•Š๊ฑฐ๋‚˜, ํŠน์ • ๊ฐ€์ƒํ™˜๊ฒฝ์˜ ์‹คํ–‰ ํŒŒ์ผ์„ ์ˆ˜๋™์œผ๋กœ ์ง€์ •ํ•˜๊ณ  ์‹ถ๋‹ค๋ฉด ํ™˜๊ฒฝ๋ณ€์ˆ˜๋ฅผ ์„ค์ •ํ•˜์„ธ์š”: +์ €์žฅ์†Œ ํ…Œ์ŠคํŠธ๋ฅผ ์‹คํ–‰ํ•ฉ๋‹ˆ๋‹ค. ```bash -# newsdom_api/mineru_runner.py ์—์„œ ์ด ํ™˜๊ฒฝ๋ณ€์ˆ˜๋ฅผ ์šฐ์„  ํƒ์ƒ‰ํ•ฉ๋‹ˆ๋‹ค. -export NEWSDOM_MINERU_BIN="/path/to/custom/mineru" +uv run pytest ``` ---- - -## 3. ์„ค์น˜ ํ™•์ธ ๋ฐ ์ƒํƒœ ์ ๊ฒ€ - -์„ค์น˜๊ฐ€ ์ •์ƒ์ ์œผ๋กœ ์™„๋ฃŒ๋˜์—ˆ๋Š”์ง€ ํ™•์ธํ•˜๋ ค๋ฉด ๊ธฐ๋ณธ ํ…Œ์ŠคํŠธ๋ฅผ ๊ตฌ๋™ํ•ด๋ณด์„ธ์š”. +API shell์€ parser ์„ค์น˜ ์—†์ด๋„ liveness, ์ธ์ฆ ์„ค์ •, ์š”์ฒญ ๊ณ„์•ฝ์„ ๊ฐœ๋ฐœยท๊ฒ€์ฆํ•˜๋Š” ๋ฐ ์‚ฌ์šฉํ•  ์ˆ˜ ์žˆ์Šต๋‹ˆ๋‹ค. ```bash -# ํŒŒ์ด์ฌ ๊ฒฝ๊ณ (Warning)๋ฅผ ์—๋Ÿฌ๋กœ ์ทจ๊ธ‰ํ•˜์—ฌ ๊ผผ๊ผผํ•˜๊ฒŒ ๊ฒ€์‚ฌ -PYTHONWARNINGS=error pytest +uv run uvicorn --app-dir src newsdom_api.main:app --host 0.0.0.0 --port 8000 --reload ``` -ํ˜„์žฌ ์ €์žฅ์†Œ์—๋Š” ๋ณ„๋„์˜ `integration` ๋งˆ์ปค ํ…Œ์ŠคํŠธ ๋ฌถ์Œ์ด ์—†์œผ๋ฏ€๋กœ, -์„ค์น˜ ํ™•์ธ์˜ ๊ธฐ์ค€์€ ๊ธฐ๋ณธ `pytest` ์Šค์œ„ํŠธ ํ†ต๊ณผ์ž…๋‹ˆ๋‹ค. ์ถ”๊ฐ€๋กœ MinerU -๊ฒฝ๋กœ์™€ API ๋™์ž‘๊นŒ์ง€ ํ™•์ธํ•˜๋ ค๋ฉด ์„œ๋ฒ„๋ฅผ ์ง์ ‘ ๋„์šด ๋’ค ์ˆ˜๋™ API ์ ๊ฒ€ -๋‹จ๊ณ„๋ฅผ ์ˆ˜ํ–‰ํ•˜์„ธ์š”. +๋‹ค๋ฅธ ํ„ฐ๋ฏธ๋„์—์„œ liveness๋ฅผ ํ™•์ธํ•ฉ๋‹ˆ๋‹ค. ```bash -# ๋ณ„๋„ ํ„ฐ๋ฏธ๋„์—์„œ API ์„œ๋ฒ„ ๊ธฐ๋™ -python -m uvicorn --app-dir src newsdom_api.main:app --host 0.0.0.0 --port 8000 --reload - -# ๋‹ค๋ฅธ ํ„ฐ๋ฏธ๋„์—์„œ ์ƒํƒœ ํ™•์ธ curl -sS http://127.0.0.1:8000/health ``` -์ •์ƒ ์‘๋‹ต ์˜ˆ์‹œ๋Š” `{"status": "ok"}`์ด๋ฉฐ HTTP 200 ์ƒํƒœ ์ฝ”๋“œ๋ฅผ ๋ฐ˜ํ™˜ํ•ด์•ผ ํ•ฉ๋‹ˆ๋‹ค. +์ •์ƒ ํ”„๋กœ์„ธ์Šค๋Š” HTTP 200์„ ๋ฐ˜ํ™˜ํ•ฉ๋‹ˆ๋‹ค. `/health` ์„ฑ๊ณต์€ parser readiness๋ฅผ ์˜๋ฏธํ•˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ์‹ค์ œ ํŠธ๋ž˜ํ”ฝ ๋ผ์šฐํŒ… ํŒ๋‹จ์—๋Š” `/ready`๋ฅผ ์‚ฌ์šฉํ•˜๊ณ , ์Šน์ธ๋œ parser๊ฐ€ ์—†๋Š” ๋™์•ˆ์—๋Š” fail-closed ์ƒํƒœ๋ฅผ ์œ ์ง€ํ•ฉ๋‹ˆ๋‹ค. -๋ชจ๋“  ํ…Œ์ŠคํŠธ(`tests/`)๊ฐ€ ์„ฑ๊ณต์ ์œผ๋กœ ํ†ต๊ณผํ–ˆ๋‹ค๋ฉด API ์„œ๋ฒ„๋ฅผ ์‹คํ–‰ํ•  -์ค€๋น„๊ฐ€ ๋œ ๊ฒƒ์ž…๋‹ˆ๋‹ค. +## 4. ๋‹ค์Œ ๋‹จ๊ณ„ -๐Ÿ‘‰ ๋‹ค์Œ ๋‹จ๊ณ„: **[API ๋ ˆํผ๋Ÿฐ์Šค ๋ฐ ์‚ฌ์šฉ ๋ฐฉ๋ฒ•](api-reference.md)** +- [API ๋ ˆํผ๋Ÿฐ์Šค ๋ฐ ์‚ฌ์šฉ ๋ฐฉ๋ฒ•](api-reference.md) +- [๊ฐœ๋ฐœ ๋ฐ ๊ธฐ์—ฌ](development.md) +- [Commercial parser replacement #671](https://github.com/ContextualWisdomLab/newsdom-api/issues/671) +- [GitHub repository README](https://github.com/ContextualWisdomLab/newsdom-api) From f0edda04080f14d54c6fed9f236d5c5ee981b3c5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 10:11:53 +0900 Subject: [PATCH 58/75] test: enforce commercial parser documentation boundary --- tests/test_manual_docs.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/test_manual_docs.py b/tests/test_manual_docs.py index aa72a2da..cdb63dec 100644 --- a/tests/test_manual_docs.py +++ b/tests/test_manual_docs.py @@ -50,20 +50,23 @@ def test_development_doc_adds_blank_line_after_branch_rules_heading() -> None: assert _has_blank_line_after_heading(text, "### ๐ŸŒฟ ๋ธŒ๋žœ์น˜ ๊ทœ์น™") -def test_installation_doc_uses_quoted_extras_and_clear_python_wording(): +def test_installation_doc_uses_locked_repo_setup_and_blocks_restricted_parser_install(): text = Path("manual/installation.md").read_text(encoding="utf-8") assert "Required: `>=3.10, <3.14`" in text for token in ("python3.10", "์ง€์› ๋ฒ”์œ„", "์ธํ„ฐํ”„๋ฆฌํ„ฐ"): assert token in text assert "python3.10 -m venv .venv" in text assert 'pip install -e ".[dev]"' in text - assert 'pip install "mineru[pipeline]==3.4.4"' in text + assert "uv sync --frozen --all-extras" in text + assert 'pip install "mineru[pipeline]==3.4.4"' not in text + assert "#671" in text + assert "์ƒ์—… ๋ฐฐํฌ ์Šน์ธ ๊ฒฝ๋กœ๊ฐ€ ์•„๋‹™๋‹ˆ๋‹ค" in text def test_installation_doc_includes_manual_api_healthcheck_commands(): text = Path("manual/installation.md").read_text(encoding="utf-8") for token in [ - "python -m uvicorn", + "uv run uvicorn", "--app-dir src", "newsdom_api.main:app", "--host 0.0.0.0", From 2bc960192cbae4a8802c7b082e4db4273a4634f6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 10:12:12 +0900 Subject: [PATCH 59/75] docs: refresh NewsDOM public manual landing --- manual/index.md | 61 +++++++++++++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 25 deletions(-) diff --git a/manual/index.md b/manual/index.md index b808afa1..e221e7d6 100644 --- a/manual/index.md +++ b/manual/index.md @@ -1,40 +1,51 @@ -# NewsDOM API ๊ฐœ์š” ๋ฐ ์•„ํ‚คํ…์ฒ˜ +# NewsDOM API -[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/Seongho-Bae/newsdom-api/badge)](https://securityscorecards.dev/viewer/?uri=github.com/Seongho-Bae/newsdom-api) +[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/ContextualWisdomLab/newsdom-api/badge)](https://securityscorecards.dev/viewer/?uri=github.com/ContextualWisdomLab/newsdom-api) +[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/ContextualWisdomLab/newsdom-api) -**NewsDOM API**๋Š” ์Šค์บ”๋œ ์ผ๋ณธ์–ด ์‹ ๋ฌธ PDF ๋ฌธ์„œ๋ฅผ ๋ถ„์„ํ•˜์—ฌ, ์›น ๋ธŒ๋ผ์šฐ์ €์˜ DOM(Document Object Model)๊ณผ ์œ ์‚ฌํ•œ ๊ธฐ์‚ฌ(Article) ๋‹จ์œ„์˜ ํŠธ๋ฆฌ ๊ตฌ์กฐ๋กœ ํŒŒ์‹ฑํ•ด์ฃผ๋Š” API ์„œ๋น„์Šค์ž…๋‹ˆ๋‹ค. +**NewsDOM API**๋Š” PDF๋ฅผ ํŽ˜์ด์ง€ยท์„น์…˜ยท์ œ๋ชฉยท๋ณธ๋ฌธ ๋ธ”๋กยท์ด๋ฏธ์ง€ยท์บก์…˜ยท๋ฐ”์šด๋”ฉ ๋ฐ•์Šค๊ฐ€ ์žˆ๋Š” ์•ˆ์ •์ ์ธ DOMํ˜• JSON์œผ๋กœ ์ •๊ทœํ™”ํ•˜๋Š” API boundary์ž…๋‹ˆ๋‹ค. ํ˜ธ์ถœ์ž๋Š” ํŠน์ • parser ๋‚ด๋ถ€ ๊ตฌ์กฐ๋ฅผ ํ•ด์„ํ•˜๋Š” ๋Œ€์‹  NewsDOM์˜ HTTPยทschemaยท์•ˆ์ „ ๊ณ„์•ฝ์„ ์‚ฌ์šฉํ•ฉ๋‹ˆ๋‹ค. -๊ณผ๊ฑฐ์˜ ์‹ ๋ฌธ ์ด๋ฏธ์ง€๋Š” ๋‹จ์ˆœ ํ…์ŠคํŠธ ์ถ”์ถœ(OCR)๋งŒ์œผ๋กœ๋Š” ๋‹ค๋‹จ ๋ ˆ์ด์•„์›ƒ์ด๋‚˜ ์ด๋ฏธ์ง€/์บก์…˜, ๊ธฐ์‚ฌ๋ณ„ ํ๋ฆ„์„ ํŒŒ์•…ํ•˜๊ธฐ ๋งค์šฐ ์–ด๋ ต์Šต๋‹ˆ๋‹ค. ๋ณธ ํ”„๋กœ์ ํŠธ๋Š” ๋”ฅ๋Ÿฌ๋‹ ๊ธฐ๋ฐ˜ ๋ ˆ์ด์•„์›ƒ ๋ถ„์„ ๋„๊ตฌ์ธ **MinerU**๋ฅผ ๋ฐฑ์—”๋“œ๋กœ ์‚ฌ์šฉํ•˜์—ฌ ์ด ๋ฌธ์ œ๋ฅผ ํ•ด๊ฒฐํ•ฉ๋‹ˆ๋‹ค. +์ผ๋ณธ์–ด ์‹ ๋ฌธ OCR์€ ์ง€์›ํ•ด ์˜จ ์‚ฌ๋ก€ ์ค‘ ํ•˜๋‚˜์ด์ง€๋งŒ ์ œํ’ˆ ๋ฒ”์œ„๋Š” ํŠน์ • ์–ธ์–ด ๋˜๋Š” ๊ฐ•์ œ OCR์— ํ•œ์ •๋˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ํ˜„์žฌ API ๊ณ„์•ฝ์€ `language` ์„ ํƒ๊ณผ `mode=auto | ocr | txt`๋ฅผ ๊ตฌ๋ถ„ํ•˜๊ณ , `/parse`์˜ fail-closed ์ธ์ฆ๊ณผ `/health`ยท`/ready`์˜ ์„œ๋กœ ๋‹ค๋ฅธ ์˜๋ฏธ๋ฅผ ์œ ์ง€ํ•ฉ๋‹ˆ๋‹ค. ---- +## ํ˜„์žฌ commercial parser ์ƒํƒœ -## โš™๏ธ ์‹œ์Šคํ…œ ๋‚ด๋ถ€ ์•„ํ‚คํ…์ฒ˜ (Under the Hood) +NewsDOM ์ž์ฒด ์†Œ์Šค๋Š” MIT์ž…๋‹ˆ๋‹ค. ๊ทธ๋Ÿฌ๋‚˜ ํ˜„์žฌ ์†Œ์Šค์˜ legacy MinerU adapter๊ฐ€ ์‚ฌ์šฉํ•˜๋Š” MinerU 3.x๋Š” Apache-2.0์— ์ถ”๊ฐ€ commercial ์กฐ๊ฑด์„ ๋‘” MinerU Open Source License๋ฅผ ์‚ฌ์šฉํ•˜๋ฏ€๋กœ ContextualWisdomLab์˜ unrestricted commercial-inbound ์ •์ฑ…๊ณผ ํ˜ธํ™˜๋˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. -์‚ฌ์šฉ์ž๊ฐ€ API๋ฅผ ํ†ตํ•ด PDF ํŒŒ์ผ์„ ์—…๋กœ๋“œํ•˜๋ฉด, NewsDOM ๋‚ด๋ถ€์—์„œ๋Š” ๋‹ค์Œ์˜ ์„ธ ๋‹จ๊ณ„๋ฅผ ๊ฑฐ์ณ ๋ฐ์ดํ„ฐ๋ฅผ ์ฒ˜๋ฆฌํ•ฉ๋‹ˆ๋‹ค: +๋”ฐ๋ผ์„œ MinerU package ์„ค์น˜, customer-supplied MinerU binary, `Dockerfile.nvidia` ๊ธฐ๋ฐ˜ ๋ฒˆ๋“ค ๋ฐฐํฌ๋ฅผ **์Šน์ธ๋œ ์ƒ์—… ๊ฒฝ๋กœ๋กœ ์•ˆ๋‚ดํ•˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค**. ๋Œ€์ฒด parser boundary์™€ ๊ฒ€์ฆ ๊ธฐ์ค€์€ [#671](https://github.com/ContextualWisdomLab/newsdom-api/issues/671)์—์„œ ๊ด€๋ฆฌํ•ฉ๋‹ˆ๋‹ค. ์Šน์ธ๋œ parser๊ฐ€ ์—†์„ ๋•Œ `/ready`๊ฐ€ fail-closed์ธ ๊ฒƒ์€ ์ •์ƒ์ž…๋‹ˆ๋‹ค. -### 1. ์„œ๋น„์Šค ๋ž˜ํผ ๋ ˆ์ด์–ด (`src/newsdom_api/service.py`) -FastAPI ์—”๋“œํฌ์ธํŠธ(`/parse`)๊ฐ€ `UploadFile`๋กœ ์ „๋‹ฌ๋ฐ›์€ ๋ฐ”์ด๋„ˆ๋ฆฌ ๋ฐ์ดํ„ฐ๋ฅผ ์ž„์‹œ ๋””๋ ‰ํ† ๋ฆฌ(Temporary Directory)์— ์ €์žฅํ•œ ํ›„, ํŒŒ์ดํ”„๋ผ์ธ ๋Ÿฌ๋„ˆ๋ฅผ ํ˜ธ์ถœํ•ฉ๋‹ˆ๋‹ค. +## ์ œํ’ˆ ๊ฒฝ๊ณ„ -### 2. MinerU ํŒŒ์ดํ”„๋ผ์ธ ๋Ÿฌ๋„ˆ (`src/newsdom_api/mineru_runner.py`) -์ €์žฅ๋œ PDF๋ฅผ ๋Œ€์ƒ์œผ๋กœ Python `subprocess` ๋ชจ๋“ˆ์„ ์ด์šฉํ•ด **MinerU CLI**๋ฅผ ๋ฐฑ๊ทธ๋ผ์šด๋“œ์—์„œ ์‹คํ–‰ํ•ฉ๋‹ˆ๋‹ค. ์‹ค์ œ ๋‚ด๋ถ€์ ์œผ๋กœ ์‹คํ–‰๋˜๋Š” ๋ช…๋ น์–ด๋Š” ๋‹ค์Œ๊ณผ ๊ฐ™์Šต๋‹ˆ๋‹ค: +- `POST /parse`๋Š” PDF ์—…๋กœ๋“œ์™€ canonical NewsDOM response ๊ณ„์•ฝ์„ ์†Œ์œ ํ•ฉ๋‹ˆ๋‹ค. +- `language`๊ณผ `mode`๋Š” request compatibility surface์ด๋ฉฐ parser implementation detail๊ณผ ๋ถ„๋ฆฌ๋˜์–ด์•ผ ํ•ฉ๋‹ˆ๋‹ค. +- `/parse`๋Š” production profile์—์„œ fail-closed Bearer ์ธ์ฆ์„ ์š”๊ตฌํ•ฉ๋‹ˆ๋‹ค. +- `GET /health`๋Š” process liveness, `GET /ready`๋Š” ์‹ค์ œ traffic readiness์ž…๋‹ˆ๋‹ค. +- parser admission์€ process๋ณ„ bounded non-waiting lease๋ฅผ ์‚ฌ์šฉํ•˜๋ฉฐ ํฌํ™” ์‹œ body ์ฒ˜๋ฆฌ ์ „์— `429`๋กœ ๊ฑฐ๋ถ€ํ•ฉ๋‹ˆ๋‹ค. +- ์ž„์‹œ workspace, parser failure ๋ถ„๋ฅ˜, schema validation, fixture provenance๋Š” NewsDOM์ด ์†Œ์œ ํ•ฉ๋‹ˆ๋‹ค. -```bash -mineru -p <์—…๋กœ๋“œ๋œ_PDF> -o <์ž„์‹œ์ถœ๋ ฅ๊ฒฝ๋กœ> -b pipeline -m ocr -l japan +## ์š”์ฒญ ํ๋ฆ„ + +```text +client + -> FastAPI request/auth boundary + -> bounded parser admission + -> request-scoped temporary PDF workspace + -> approved parser adapter port + -> NewsDOM normalization + schema validation + -> canonical JSON response ``` -> *์ฐธ๊ณ : ์ผ๋ณธ์–ด ์‹ ๋ฌธ ์ฒ˜๋ฆฌ์— ์ตœ์ ํ™”ํ•˜๊ธฐ ์œ„ํ•ด `-l japan` (Language: Japanese) ์˜ต์…˜๊ณผ OCR ํŒŒ์ดํ”„๋ผ์ธ ๋ชจ๋“œ๊ฐ€ ํ•˜๋“œ์ฝ”๋”ฉ๋˜์–ด ์žˆ์Šต๋‹ˆ๋‹ค.* -๋ช…๋ น์–ด ์‹คํ–‰์ด ์™„๋ฃŒ๋˜๋ฉด ๋Ÿฌ๋„ˆ๋Š” ์ƒ์„ฑ๋œ ์ถœ๋ ฅ ํด๋”(OCR ํ•˜์œ„ ํด๋”)๋ฅผ ๋’ค์ ธ `*_content_list.json` ํŒŒ์ผ๊ณผ `*_model.json` ๊ฒฐ๊ณผ๋ฌผ์„ ๋ฉ”๋ชจ๋ฆฌ๋กœ ๋กœ๋“œํ•ฉ๋‹ˆ๋‹ค. +ํ˜„์žฌ legacy adapter๋Š” ๊ต์ฒด ๋Œ€์ƒ์ž…๋‹ˆ๋‹ค. downstream consumer๋Š” parser-specific type์ด๋‚˜ model identity๊ฐ€ ์•„๋‹ˆ๋ผ NewsDOM contract์— ์˜์กดํ•ด์•ผ ํ•ฉ๋‹ˆ๋‹ค. + +## ์‹œ์ž‘ํ•˜๊ธฐ -### 3. DOM ๋นŒ๋” (`src/newsdom_api/dom_builder.py`) -MinerU๊ฐ€ ๋ฑ‰์–ด๋‚ธ ์„ ํ˜•์ ์ธ(Linear) ๋ธ”๋ก ๋ฆฌ์ŠคํŠธ(`content_list.json`)๋ฅผ ์ˆœํšŒํ•˜๋ฉด์„œ ๋…ผ๋ฆฌ์ ์ธ ํŠธ๋ฆฌ ํ˜•ํƒœ์˜ **`ParseResponse` (Canonical JSON)**๋กœ ์žฌ๊ตฌ์„ฑํ•ฉ๋‹ˆ๋‹ค. -- `role == "header"` ์ด๋ฉด ์ƒ๋‹จ ๋จธ๋ฆฟ๋ง๋กœ(`PageNode.headers`) ๋ถ„๋ฅ˜ -- `type == "ad"` ๋˜๋Š” `role == "ad"` ์ด๋ฉด ์ง€๋ฉด ๊ด‘๊ณ (`PageNode.ads`)๋กœ ๋ถ„๋ฅ˜ -- `text_level == 1` ์ด๊ฑฐ๋‚˜ `role == "section_headings"`์ธ ๊ฒฝ์šฐ ์ƒˆ๋กœ์šด ๊ธฐ์‚ฌ์˜ ์‹œ์ž‘(Headline)์œผ๋กœ ์ธ์‹ํ•˜์—ฌ ์ƒˆ `ArticleNode`๋ฅผ ์ƒ์„ฑ -- ์ดํ›„ ๋“ฑ์žฅํ•˜๋Š” ์ผ๋ฐ˜ ํ…์ŠคํŠธ๋Š” ํ•ด๋‹น ๊ธฐ์‚ฌ์˜ `body_blocks` ๋ฐฐ์—ด์— ์ถ”๊ฐ€ -- `type == "image"` ์ด๋ฉด `ImageNode`๋ฅผ ์ƒ์„ฑํ•˜๊ณ  ํฌํ•จ๋œ ์บก์…˜ ๋ฐฐ์—ด์„ ํŒŒ์‹ฑํ•˜์—ฌ ๊ธฐ์‚ฌ์— ์ข…์†์‹œํ‚ด +- [์„ค์น˜ ๊ฐ€์ด๋“œ](installation.md) โ€” NewsDOM ์ž์ฒด์˜ lockfile ๊ธฐ๋ฐ˜ ๊ฐœ๋ฐœ/๊ฒ€์ฆ ํ™˜๊ฒฝ๊ณผ parser license boundary. +- [์‚ฌ์šฉ ๋ฐฉ๋ฒ• ๋ฐ API](api-reference.md) โ€” `/parse`, language/mode, ์ธ์ฆ, ์‘๋‹ต๊ณผ ์˜ค๋ฅ˜ ๊ณ„์•ฝ. +- [๊ฐœ๋ฐœ ๋ฐ ๊ธฐ์—ฌ](development.md) โ€” ๋กœ์ปฌ ๊ฒ€์ฆ๊ณผ ๊ธฐ์—ฌ ํ๋ฆ„. +- [GitHub ์ €์žฅ์†Œ](https://github.com/ContextualWisdomLab/newsdom-api) โ€” README, architecture, security, releases, source history. +- [Commercial parser replacement #671](https://github.com/ContextualWisdomLab/newsdom-api/issues/671) โ€” ํ˜„์žฌ ์ƒ์—… ๋ผ์ด์„ ์Šค ์ฐจ๋‹จ๊ณผ replacement acceptance. -์ด๋Ÿฌํ•œ ์„ธ๋ฐ€ํ•œ ๋‚ด๋ถ€ ๋ณ€ํ™˜ ๊ณผ์ •์„ ํ†ตํ•ด ๋‹จ์ˆœํ•œ OCR ํ…์ŠคํŠธ ๋คํ”„๊ฐ€ ์•„๋‹Œ, ํ”„๋ก ํŠธ์—”๋“œ์—์„œ ์ฆ‰์‹œ ๋ Œ๋”๋ง์ด ๊ฐ€๋Šฅํ•œ **๊ตฌ์กฐํ™”๋œ DOM ๋ฐ์ดํ„ฐ**๊ฐ€ ์ตœ์ข… ๋ฐ˜ํ™˜๋ฉ๋‹ˆ๋‹ค. +## ๋ฆด๋ฆฌ์Šค์™€ ๊ณต๊ฐœ ๋ฌธ์„œ ---- +`pyproject.toml`์˜ version์€ source metadata์ด๋ฉฐ ๊ทธ ์ž์ฒด๋กœ immutable release evidence๊ฐ€ ์•„๋‹™๋‹ˆ๋‹ค. ์‹ค์ œ ๋ฆด๋ฆฌ์Šค๋Š” [GitHub Releases](https://github.com/ContextualWisdomLab/newsdom-api/releases)์™€ ํ•ด๋‹น exact-head artifact/provenance evidence๋กœ ํ™•์ธํ•ฉ๋‹ˆ๋‹ค. -๐Ÿ‘‰ **[์„ค์น˜ ๊ฐ€์ด๋“œ](installation.md)**๋ฅผ ์ฝ๊ณ  ์ง์ ‘ ํ™˜๊ฒฝ์„ ๊ตฌ์„ฑํ•ด ๋ณด์„ธ์š”. \ No newline at end of file +์ด `manual/` source๊ฐ€ ๋ณ‘ํ•ฉ๋˜์—ˆ๋‹ค๋Š” ์‚ฌ์‹ค๋งŒ์œผ๋กœ ๊ณต๊ฐœ ์‚ฌ์ดํŠธ ๋ฐฐํฌ๊ฐ€ ์™„๋ฃŒ๋œ ๊ฒƒ์€ ์•„๋‹™๋‹ˆ๋‹ค. GitHub Pages workflow ์„ฑ๊ณต๊ณผ live HTTPS content ์žฌํ™•์ธ์ด ํ•„์š”ํ•ฉ๋‹ˆ๋‹ค. From 35ad4e905c8a11f6b60fe9a5d0c0442b8842a682 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 10:12:21 +0900 Subject: [PATCH 60/75] docs: correct organization Pages metadata --- mkdocs.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mkdocs.yml b/mkdocs.yml index e2489723..4649b793 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,8 +1,8 @@ site_name: NewsDOM API ๋งค๋‰ด์–ผ -site_url: https://seongho-bae.github.io/newsdom-api/ -site_description: ์ผ๋ณธ์–ด ์‹ ๋ฌธ ์Šค์บ” PDF๋ฅผ DOM ํ˜•ํƒœ์˜ ํŠธ๋ฆฌ๋กœ ํŒŒ์‹ฑํ•˜๋Š” API -repo_url: https://github.com/Seongho-Bae/newsdom-api -repo_name: Seongho-Bae/newsdom-api +site_url: https://contextualwisdomlab.github.io/newsdom-api/ +site_description: ๋‹ค๊ตญ์–ด PDF๋ฅผ ๊ตฌ์กฐํ™”๋œ DOMํ˜• JSON์œผ๋กœ ๋ณ€ํ™˜ํ•˜๋Š” NewsDOM API ๋ฌธ์„œ +repo_url: https://github.com/ContextualWisdomLab/newsdom-api +repo_name: ContextualWisdomLab/newsdom-api docs_dir: manual From 042b5062166dc6392434ced7106eb52a7b66c52b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 12:36:12 +0900 Subject: [PATCH 61/75] docs(security): use organization advisory route --- SECURITY.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index e80f8ecf..c63654e9 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,13 +2,17 @@ ## Reporting a vulnerability -Please report a vulnerability privately by opening a GitHub Security -Advisory draft for this repository: +Please report a vulnerability privately through this repository's GitHub +Security Advisory workflow when it is available: -- [GitHub Security Advisory draft](https://github.com/Seongho-Bae/newsdom-api/security/advisories/new) -- [Maintainer profile](https://github.com/seonghobae) +- [GitHub Security Advisory draft](https://github.com/ContextualWisdomLab/newsdom-api/security/advisories/new) -Do not open a public issue for an unpatched vulnerability. +If that private repository feature is unavailable to you, contact the +ContextualWisdomLab repository maintainers through an established private +channel rather than disclosing the issue publicly. + +Do not open a public issue, pull-request comment, or discussion for an +unpatched vulnerability. When reporting, include: From 4824d880101c53cdc11e8104181ece60a71c6438 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 19 Sep 2026 22:04:30 +0900 Subject: [PATCH 62/75] chore(repo): remove inherited one-shot repair script Protected develop already retired fix_pr.sh after proving its task-local caller obsolete. Preserve that deletion while keeping the #548 product delta stacked on #822. --- fix_pr.sh | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 fix_pr.sh diff --git a/fix_pr.sh b/fix_pr.sh deleted file mode 100644 index d4434a88..00000000 --- a/fix_pr.sh +++ /dev/null @@ -1,4 +0,0 @@ -# We don't actually push to remote directly, we just make a commit locally -# since `submit` handles the commit message logic and branch name. - -git checkout -b bolt/defaultdict-optimization || git checkout bolt/defaultdict-optimization From bb7474c518da6cb85d4b3f44e55fcf61f0e6f73f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 20 Sep 2026 00:04:08 +0900 Subject: [PATCH 63/75] test(benchmark): require reproducible bounded fixtures --- tests/test_upload_ingestion_benchmark.py | 58 ++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/tests/test_upload_ingestion_benchmark.py b/tests/test_upload_ingestion_benchmark.py index f67fc44f..70365aab 100644 --- a/tests/test_upload_ingestion_benchmark.py +++ b/tests/test_upload_ingestion_benchmark.py @@ -11,6 +11,22 @@ from tools import benchmark_upload_ingestion as benchmark +def _environment_manifest() -> dict[str, int | str]: + """Return complete host evidence for a deterministic benchmark report.""" + + return { + "execution_image": "host:ubuntu-24.04", + "cpu_model": "Test CPU", + "memory_bytes": 8 * 1024**3, + "storage_device": "test-volume", + "filesystem": "ext4", + "dependency_lock_sha256": "sha256:" + "a" * 64, + "commit_sha": "b" * 40, + "worker_count": 1, + "cache_state": "cold", + } + + def _sample( duration: float = 0.25, *, @@ -78,6 +94,33 @@ def test_fixture_inventory_rejects_non_pdf_magic(tmp_path: Path) -> None: benchmark.inventory_fixture(fixture) +@pytest.mark.parametrize( + ("size_bytes", "accepted"), + [ + (20 * 1024 * 1024, True), + (20 * 1024 * 1024 + 1, False), + ], +) +def test_fixture_inventory_enforces_twenty_mib_boundary( + tmp_path: Path, + size_bytes: int, + accepted: bool, +) -> None: + """The fixture contract accepts the boundary and rejects oversized input.""" + + fixture = tmp_path / "boundary.pdf" + with fixture.open("wb") as fixture_handle: + fixture_handle.write(b"%PDF-") + fixture_handle.seek(size_bytes - 1) + fixture_handle.write(b"\0") + + if accepted: + assert benchmark.inventory_fixture(fixture)["size_bytes"] == size_bytes + else: + with pytest.raises(ValueError, match="20 MiB"): + benchmark.inventory_fixture(fixture) + + def test_fixture_inventory_records_hash_and_size(tmp_path: Path) -> None: """Every benchmark case should be traceable to immutable fixture bytes.""" @@ -217,6 +260,7 @@ async def fake_cohort( candidates=("8kib", "adaptive"), concurrency_levels=(1, 2), repetitions=2, + environment_manifest=_environment_manifest(), cohort_runner=fake_cohort, ) @@ -228,6 +272,8 @@ async def fake_cohort( ) + "\n" assert report["schema_version"] == "1.1.0" assert report["benchmark_environment"]["python_version"] + for field_name, field_value in _environment_manifest().items(): + assert report["benchmark_environment"][field_name] == field_value assert len(report["fixtures"]) == 1 assert len(report["cases"]) == 4 assert len(report["cases"][0]["samples"]) == 2 @@ -260,11 +306,16 @@ def test_raw_evidence_schema_is_strict_and_covers_required_metrics() -> None: schema = json.loads(schema_path.read_text(encoding="utf-8")) case_schema = schema["$defs"]["benchmark_case"] metric_properties = case_schema["properties"]["metrics"]["properties"] + environment_schema = schema["$defs"]["benchmark_environment"] + required_environment_fields = set(_environment_manifest()) assert schema["$schema"] == "https://json-schema.org/draft/2020-12/schema" assert schema["properties"]["schema_version"] == {"const": "1.1.0"} assert schema["additionalProperties"] is False assert case_schema["additionalProperties"] is False + assert environment_schema["additionalProperties"] is False + assert required_environment_fields <= set(environment_schema["required"]) + assert required_environment_fields <= set(environment_schema["properties"]) assert "cohort_observations" in case_schema["required"] assert case_schema["properties"]["concurrency"] == { "type": "integer", @@ -292,6 +343,8 @@ def test_cli_defaults_to_the_complete_evidence_matrix(monkeypatch, tmp_path: Pat fixtures = tmp_path / "fixtures" fixtures.mkdir() output = tmp_path / "results.json" + manifest_path = tmp_path / "environment.json" + manifest_path.write_text(json.dumps(_environment_manifest()), encoding="utf-8") captured: dict[str, object] = {} async def fake_run_matrix( @@ -301,6 +354,7 @@ async def fake_run_matrix( candidates: tuple[str, ...], concurrency_levels: tuple[int, ...], repetitions: int, + environment_manifest: dict[str, int | str], cohort_runner=benchmark.run_cohort, ) -> dict[str, object]: captured.update( @@ -309,6 +363,7 @@ async def fake_run_matrix( candidates=candidates, concurrency_levels=concurrency_levels, repetitions=repetitions, + environment_manifest=environment_manifest, cohort_runner=cohort_runner, ) return {} @@ -322,6 +377,8 @@ async def fake_run_matrix( str(fixtures), "--output", str(output), + "--environment-manifest", + str(manifest_path), "--repetitions", "3", ] @@ -331,4 +388,5 @@ async def fake_run_matrix( assert captured["concurrency_levels"] == benchmark.CONCURRENCY_LEVELS assert captured["repetitions"] == 3 assert captured["output_path"] == output + assert captured["environment_manifest"] == _environment_manifest() assert asyncio.iscoroutinefunction(fake_run_matrix) From a370d2e69826708d0cc8b5fc79028e3c033431af Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 20 Sep 2026 00:04:23 +0900 Subject: [PATCH 64/75] test(readiness): reject unapproved legacy parser --- tests/test_auth.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_auth.py b/tests/test_auth.py index 1bc71935..83db35f7 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -304,6 +304,20 @@ def test_default_runtime_probe_is_not_an_unconditional_success( assert response.status_code == 503 +def test_legacy_mineru_executable_is_not_an_approved_parser( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A discoverable legacy executable cannot make commercial traffic ready.""" + + monkeypatch.setattr( + "newsdom_api.main.mineru_runtime_available", lambda: True, raising=False + ) + response = TestClient(create_app(_settings())).get("/ready") + + assert response.status_code == 503 + assert response.json() == {"detail": "Service Unavailable"} + + def test_authentication_middleware_rejects_before_reading_request_body() -> None: """Reject parser access before ASGI body receive or downstream parsing.""" From f8efe441f7343c328760bddf115c8735b461e086 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 20 Sep 2026 00:04:35 +0900 Subject: [PATCH 65/75] test(benchmark): require manifest schema revision --- tests/test_upload_ingestion_benchmark.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_upload_ingestion_benchmark.py b/tests/test_upload_ingestion_benchmark.py index 70365aab..1be8ddc5 100644 --- a/tests/test_upload_ingestion_benchmark.py +++ b/tests/test_upload_ingestion_benchmark.py @@ -270,7 +270,7 @@ async def fake_cohort( indent=2, sort_keys=True, ) + "\n" - assert report["schema_version"] == "1.1.0" + assert report["schema_version"] == "1.2.0" assert report["benchmark_environment"]["python_version"] for field_name, field_value in _environment_manifest().items(): assert report["benchmark_environment"][field_name] == field_value @@ -310,7 +310,7 @@ def test_raw_evidence_schema_is_strict_and_covers_required_metrics() -> None: required_environment_fields = set(_environment_manifest()) assert schema["$schema"] == "https://json-schema.org/draft/2020-12/schema" - assert schema["properties"]["schema_version"] == {"const": "1.1.0"} + assert schema["properties"]["schema_version"] == {"const": "1.2.0"} assert schema["additionalProperties"] is False assert case_schema["additionalProperties"] is False assert environment_schema["additionalProperties"] is False From ed4d9341b4c92a117888a4d455e2092c363ac184 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 20 Sep 2026 00:05:06 +0900 Subject: [PATCH 66/75] fix(benchmark): bind reports to reproducible host evidence --- tools/benchmark_upload_ingestion.py | 87 ++++++++++++++++++++++++++++- 1 file changed, 84 insertions(+), 3 deletions(-) diff --git a/tools/benchmark_upload_ingestion.py b/tools/benchmark_upload_ingestion.py index 487869c0..8fd6cfa6 100644 --- a/tools/benchmark_upload_ingestion.py +++ b/tools/benchmark_upload_ingestion.py @@ -9,6 +9,7 @@ import math import os import platform +import re import sys import tempfile import time @@ -34,6 +35,22 @@ 20 * 1024 * 1024, ) _RESULT_SCHEMA_PATH = "docs/benchmarks/upload-ingestion-result.schema.json" +MAX_FIXTURE_BYTES = 20 * 1024 * 1024 +_ENVIRONMENT_MANIFEST_FIELDS = frozenset( + { + "execution_image", + "cpu_model", + "memory_bytes", + "storage_device", + "filesystem", + "dependency_lock_sha256", + "commit_sha", + "worker_count", + "cache_state", + } +) +_SHA256_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$") +_COMMIT_SHA_PATTERN = re.compile(r"^[0-9a-f]{40}$") SampleRecord = dict[str, int | float] @@ -99,12 +116,15 @@ def inventory_fixture(path: Path) -> dict[str, int | str]: if not path.is_file() or path.is_symlink(): raise ValueError(f"Fixture must be a regular non-symlink file: {path}") + size_bytes = path.stat().st_size + if size_bytes > MAX_FIXTURE_BYTES: + raise ValueError(f"Fixture exceeds 20 MiB: {path.name}") with path.open("rb") as handle: if handle.read(5) != b"%PDF-": raise ValueError(f"Fixture does not contain PDF magic bytes: {path.name}") return { "fixture_name": path.name, - "size_bytes": path.stat().st_size, + "size_bytes": size_bytes, "sha256": _sha256(path), } @@ -290,6 +310,52 @@ def _environment_record() -> dict[str, Any]: } +def _validated_environment_manifest( + manifest: dict[str, Any], +) -> dict[str, int | str]: + """Validate caller-supplied hardware and execution provenance.""" + + field_names = set(manifest) + if field_names != _ENVIRONMENT_MANIFEST_FIELDS: + missing = sorted(_ENVIRONMENT_MANIFEST_FIELDS - field_names) + unexpected = sorted(field_names - _ENVIRONMENT_MANIFEST_FIELDS) + raise ValueError( + f"Environment manifest fields mismatch; missing={missing}, " + f"unexpected={unexpected}" + ) + string_fields = ( + "execution_image", + "cpu_model", + "storage_device", + "filesystem", + ) + if any( + not isinstance(manifest[field_name], str) + or not str(manifest[field_name]).strip() + for field_name in string_fields + ): + raise ValueError("Environment manifest string fields must be non-empty") + if ( + not isinstance(manifest["memory_bytes"], int) + or manifest["memory_bytes"] < 1 + or not isinstance(manifest["worker_count"], int) + or manifest["worker_count"] < 1 + ): + raise ValueError("Environment manifest counts must be positive integers") + dependency_lock = manifest["dependency_lock_sha256"] + commit_sha = manifest["commit_sha"] + if ( + not isinstance(dependency_lock, str) + or _SHA256_PATTERN.fullmatch(dependency_lock) is None + or not isinstance(commit_sha, str) + or _COMMIT_SHA_PATTERN.fullmatch(commit_sha) is None + ): + raise ValueError("Environment manifest hashes are invalid") + if manifest["cache_state"] not in {"cold", "warm", "disabled"}: + raise ValueError("Environment manifest cache_state is invalid") + return dict(manifest) + + def aggregate_metrics( cohorts: Sequence[CohortObservation], ) -> dict[str, int | float]: @@ -338,6 +404,7 @@ async def run_matrix( candidates: tuple[str, ...] = tuple(CHUNK_CANDIDATES), concurrency_levels: tuple[int, ...] = CONCURRENCY_LEVELS, repetitions: int = 3, + environment_manifest: dict[str, Any], cohort_runner: CohortRunner = run_cohort, ) -> dict[str, Any]: """Run and write the reproducible raw upload-ingestion evidence matrix.""" @@ -347,6 +414,8 @@ async def run_matrix( if not fixture_paths: raise ValueError("At least one PDF fixture is required") + benchmark_environment = _environment_record() + benchmark_environment.update(_validated_environment_manifest(environment_manifest)) fixture_records = [inventory_fixture(path) for path in fixture_paths] cases: list[dict[str, Any]] = [] for concurrency in concurrency_levels: @@ -385,10 +454,10 @@ async def run_matrix( ) report: dict[str, Any] = { - "schema_version": "1.1.0", + "schema_version": "1.2.0", "schema_path": _RESULT_SCHEMA_PATH, "generated_at": datetime.now(timezone.utc).isoformat(), - "benchmark_environment": _environment_record(), + "benchmark_environment": benchmark_environment, "fixtures": fixture_records, "cases": cases, } @@ -429,6 +498,12 @@ def main(argv: list[str] | None = None) -> None: required=True, help="Path for the schema-versioned raw JSON report.", ) + parser.add_argument( + "--environment-manifest", + type=Path, + required=True, + help="Strict JSON hardware, lock, revision, worker, and cache evidence.", + ) parser.add_argument( "--repetitions", type=_positive_int, @@ -451,6 +526,11 @@ def main(argv: list[str] | None = None) -> None: ) args = parser.parse_args(argv) fixture_paths = discover_fixtures(args.fixtures_dir) + environment_manifest = json.loads( + args.environment_manifest.read_text(encoding="utf-8") + ) + if not isinstance(environment_manifest, dict): + raise ValueError("Environment manifest must be a JSON object") candidates = tuple(args.candidates or CHUNK_CANDIDATES) concurrency_levels = tuple(args.concurrency_levels or CONCURRENCY_LEVELS) asyncio.run( @@ -460,6 +540,7 @@ def main(argv: list[str] | None = None) -> None: candidates=candidates, concurrency_levels=concurrency_levels, repetitions=args.repetitions, + environment_manifest=environment_manifest, ) ) From 24b47f7c368699185f96d145a3dc3ce767c06a36 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 20 Sep 2026 00:05:16 +0900 Subject: [PATCH 67/75] docs(benchmark): require reproducibility manifest --- .../upload-ingestion-result.schema.json | 77 +++++++++++++++++-- 1 file changed, 71 insertions(+), 6 deletions(-) diff --git a/docs/benchmarks/upload-ingestion-result.schema.json b/docs/benchmarks/upload-ingestion-result.schema.json index 438d9681..3ec93136 100644 --- a/docs/benchmarks/upload-ingestion-result.schema.json +++ b/docs/benchmarks/upload-ingestion-result.schema.json @@ -15,7 +15,7 @@ ], "properties": { "schema_version": { - "const": "1.1.0" + "const": "1.2.0" }, "schema_path": { "const": "docs/benchmarks/upload-ingestion-result.schema.json" @@ -53,7 +53,16 @@ "platform", "machine", "processor_count", - "rss_provider" + "rss_provider", + "execution_image", + "cpu_model", + "memory_bytes", + "storage_device", + "filesystem", + "dependency_lock_sha256", + "commit_sha", + "worker_count", + "cache_state" ], "properties": { "python_version": { @@ -76,18 +85,64 @@ "type": "string" }, "processor_count": { - "type": ["integer", "null"], + "type": [ + "integer", + "null" + ], "minimum": 1 }, "rss_provider": { "const": "linux_proc_status_vmrss" + }, + "execution_image": { + "type": "string", + "minLength": 1 + }, + "cpu_model": { + "type": "string", + "minLength": 1 + }, + "memory_bytes": { + "type": "integer", + "minimum": 1 + }, + "storage_device": { + "type": "string", + "minLength": 1 + }, + "filesystem": { + "type": "string", + "minLength": 1 + }, + "dependency_lock_sha256": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "commit_sha": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, + "worker_count": { + "type": "integer", + "minimum": 1 + }, + "cache_state": { + "enum": [ + "cold", + "warm", + "disabled" + ] } } }, "fixture_record": { "type": "object", "additionalProperties": false, - "required": ["fixture_name", "size_bytes", "sha256"], + "required": [ + "fixture_name", + "size_bytes", + "sha256" + ], "properties": { "fixture_name": { "type": "string", @@ -123,7 +178,13 @@ "minLength": 1 }, "candidate": { - "enum": ["8kib", "64kib", "256kib", "1mib", "adaptive"] + "enum": [ + "8kib", + "64kib", + "256kib", + "1mib", + "adaptive" + ] }, "chunk_bytes": { "type": "integer", @@ -256,7 +317,11 @@ "sample_record": { "type": "object", "additionalProperties": false, - "required": ["duration_seconds", "read_calls", "bytes_copied"], + "required": [ + "duration_seconds", + "read_calls", + "bytes_copied" + ], "properties": { "duration_seconds": { "type": "number", From f24435695f4c648382b6191e6b9ecb8b36070b40 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 20 Sep 2026 00:05:28 +0900 Subject: [PATCH 68/75] fix(readiness): require an approved parser backend --- src/newsdom_api/main.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/newsdom_api/main.py b/src/newsdom_api/main.py index db216b35..3afaf53e 100644 --- a/src/newsdom_api/main.py +++ b/src/newsdom_api/main.py @@ -190,6 +190,16 @@ def health() -> HealthResponse: return HealthResponse() +def approved_parser_runtime_available() -> bool: + """Fail closed until an unrestricted parser backend is released.""" + + if not mineru_runtime_available(): + return False + # The discoverable MinerU 3.x adapter is legacy-only and is not approved + # for the unrestricted commercial runtime tracked by issue #671. + return False + + def ready(request: Request) -> ReadinessResponse: """Return readiness only when authentication and MinerU runtime are available.""" @@ -351,7 +361,7 @@ def create_app( application_settings.max_concurrent_parses ) application.state.runtime_readiness_probe = ( - runtime_readiness_probe or mineru_runtime_available + runtime_readiness_probe or approved_parser_runtime_available ) application.middleware("http")(security_boundary_middleware) application.add_exception_handler(Exception, global_exception_handler) @@ -372,7 +382,7 @@ def create_app( summary="Readiness Check", description=( "Returns ready only when parser authentication configuration and " - "the MinerU runtime can safely accept traffic." + "an approved parser backend can safely accept traffic." ), responses={503: {"description": SERVICE_UNAVAILABLE_DETAIL}}, tags=["System"], From 87bdd14328b1941336303cdc9dd7d17fcf3cfb9c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 20 Sep 2026 00:05:57 +0900 Subject: [PATCH 69/75] fix(config): keep environment keys ordered --- .env.example | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index fce7f0a4..4d8c18d2 100644 --- a/.env.example +++ b/.env.example @@ -1,9 +1,9 @@ # Production and shared environments use the fail-closed authentication mode. NEWSDOM_AUTH_MODE=required +# Per-process non-waiting parser admission budget. Scale replicas separately. +NEWSDOM_MAX_CONCURRENT_PARSES=1 NEWSDOM_RUNTIME_PROFILE=production # Generate a high-entropy token in the deployment secret store. Never commit it. NEWSDOM_API_TOKEN= -# Per-process non-waiting parser admission budget. Scale replicas separately. -NEWSDOM_MAX_CONCURRENT_PARSES=1 # Optional API-only image override for a bundled or mounted MinerU executable. # NEWSDOM_MINERU_BIN=/opt/mineru/bin/mineru From 9ab513aac37ac76b0d9b4513c6a64295d229f647 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 20 Sep 2026 00:05:58 +0900 Subject: [PATCH 70/75] docs(architecture): align immutable runtime settings --- ARCHITECTURE.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 8f1a7f72..c0879e0b 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -14,8 +14,7 @@ module behind an API gateway. FastAPI. Its outer middleware authenticates and admits parser work before the multipart body is read. - `src/newsdom_api/config.py` snapshots immutable runtime authentication, - profile, executable, and per-process capacity settings during application - creation. + profile, and per-process capacity settings during application creation. - `src/newsdom_api/admission.py` owns the process-local non-waiting parser lease pool. `ParseAdmissionLimiter` uses `threading.BoundedSemaphore` so excess release is treated as a programming error rather than silently inflating @@ -86,8 +85,8 @@ bounded retry semantics. ## Liveness and readiness `GET /health` proves only that the web process is live. `GET /ready` proves that -fail-closed authentication configuration and the MinerU executable are -available. Parser saturation does not make a process unready: readiness is a +fail-closed authentication configuration and an approved parser backend are +available. A discoverable legacy MinerU executable alone is not readiness. Parser saturation does not make a process unready: readiness is a routing eligibility signal, whereas the admission response is instantaneous load feedback for each request. A future durable asynchronous job API may add queue-depth readiness policy without changing this synchronous endpoint's From 96fb2d8b5ed224e9272fa996d8cb743dbc795c2c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 20 Sep 2026 00:06:00 +0900 Subject: [PATCH 71/75] docs(changelog): record approved parser readiness --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c8bfba3f..1528cba0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 > acceptance are aligned for an actual 0.3.0 publication. ### Changed + - `/parse`๋ฅผ ์–ธ์–ด ์„ ํƒํ˜• ํŒŒ์„œ๋กœ ์ผ๋ฐ˜ํ™”: MinerU `-l japan`/`-m ocr` ํ•˜๋“œ์ฝ”๋”ฉ์„ ์ œ๊ฑฐํ•˜๊ณ  optional form ํ•„๋“œ `language`(MinerU 3.4.4 ๊ณต์‹ ๊ธฐ๋ณธ `ch`, ๊ณต๊ฐœ ์–ธ์–ด๊ตฐ/alias ๊ฒ€์ฆ)์™€ `mode`(`auto`/`ocr`/`txt`, ๊ธฐ๋ณธ `auto`)๋กœ ํŒŒ๋ผ๋ฏธํ„ฐํ™”. `mode=auto`๋Š” born-digital PDF๊ฐ€ ๊ฐ•์ œ OCR์„ ๊ฑด๋„ˆ๋›ฐ๋„๋ก ํ•จ. ๊ธฐ์กด ์ž…๋ ฅ `language=japan&mode=ocr`๋Š” ๊ณต์‹ ๊ทœ์•ฝ๋Œ€๋กœ `ch`/`ocr`๋กœ ์ •๊ทœํ™”๋จ. - OpenAPI ์ œ๋ชฉ/์„ค๋ช…, README, `ArticleNode.headline` ๋ฌธ์„œ๋ฅผ ์ผ๋ฐ˜ ๋ฌธ์„œ์šฉ (section heading) ํ‘œํ˜„์œผ๋กœ ์žฌ๊ตฌ์„ฑํ•˜์—ฌ ํŠน์ • ์–ธ์–ด/์‹ ๋ฌธ ๊ฐ€์ •์„ ์†Œ๋น„์ž์—๊ฒŒ ๋…ธ์ถœํ•˜์ง€ ์•Š๋„๋ก ํ•จ. ์‘๋‹ต ์Šคํ‚ค๋งˆ ํ•„๋“œ๋Š” ํ•˜์œ„ ํ˜ธํ™˜์„ ์œ„ํ•ด ๋ณ€๊ฒฝํ•˜์ง€ ์•Š์Œ. @@ -28,9 +29,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [CLI] ํŒŒ์‹ฑ๋œ NewsDOM JSON์—์„œ ์ˆœ์ˆ˜ ํ…์ŠคํŠธ ๋ฐ์ดํ„ฐ๋ฅผ ์ถ”์ถœํ•˜์—ฌ ํ…์ŠคํŠธ ํŒŒ์ผ ๋˜๋Š” stdout์œผ๋กœ ์ถœ๋ ฅํ•˜๋Š” `tools/extract_text.py` ๋„๊ตฌ๋ฅผ ์ถ”๊ฐ€ํ–ˆ์Šต๋‹ˆ๋‹ค. ### Security + - Bounded active parser work prevents one process from starting unbounded MinerU jobs under burst traffic. Excess requests fail before body reads with the RFC 6585 429 contract and RFC 9111 no-store semantics; gateway-level tenant quotas and durable job queues remain separate controls. - `/parse` authentication is now immutable per application instance and fails closed before multipart body parsing when required configuration is missing. Hostile missing, invalid, Unicode, oversized, and duplicated Authorization headers return one non-sensitive response. -- Added unauthenticated `/ready` traffic readiness that combines authentication configuration with MinerU executable availability while `/health` remains liveness-only. +- Added unauthenticated `/ready` traffic readiness that combines authentication configuration with approved-parser availability; a legacy MinerU executable alone remains fail-closed while `/health` remains liveness-only. - Hardened the Kubernetes deployment example with a restricted namespace policy, explicit non-root UID/GID, `RuntimeDefault` seccomp, disabled privilege escalation, dropped Linux capabilities, a read-only root filesystem, and bounded writable runtime volumes. - ์ „์—ญ 500 ์—๋Ÿฌ ์‘๋‹ต์—๋„ ํ‘œ์ค€ ๋ณด์•ˆ ํ—ค๋”๋ฅผ ์ ์šฉํ•˜์—ฌ ์˜ˆ์™ธ ๊ฒฝ๋กœ์—์„œ header ๋ˆ„๋ฝ์„ ๋ฐฉ์ง€ - MinerU subprocess argv ์ƒ์„ฑ ์‹œ `-`๋กœ ์‹œ์ž‘ํ•˜๋Š” option-like ์ธ์ž๋ฅผ ๊ฑฐ๋ถ€ํ•˜์—ฌ argument injection ์œ„ํ—˜์„ ๋‚ฎ์ถค From ae27e4c328748a1180090ba504a2ba9524c760f9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 20 Sep 2026 00:06:02 +0900 Subject: [PATCH 72/75] docs(install): bind local API shell to loopback --- manual/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manual/installation.md b/manual/installation.md index e84526c4..45427c7c 100644 --- a/manual/installation.md +++ b/manual/installation.md @@ -47,7 +47,7 @@ uv run pytest API shell์€ parser ์„ค์น˜ ์—†์ด๋„ liveness, ์ธ์ฆ ์„ค์ •, ์š”์ฒญ ๊ณ„์•ฝ์„ ๊ฐœ๋ฐœยท๊ฒ€์ฆํ•˜๋Š” ๋ฐ ์‚ฌ์šฉํ•  ์ˆ˜ ์žˆ์Šต๋‹ˆ๋‹ค. ```bash -uv run uvicorn --app-dir src newsdom_api.main:app --host 0.0.0.0 --port 8000 --reload +uv run uvicorn --app-dir src newsdom_api.main:app --host 127.0.0.1 --port 8000 --reload ``` ๋‹ค๋ฅธ ํ„ฐ๋ฏธ๋„์—์„œ liveness๋ฅผ ํ™•์ธํ•ฉ๋‹ˆ๋‹ค. From 210b7a7e03c97e3b28a977234b3062f3e81dadcf Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 20 Sep 2026 00:06:13 +0900 Subject: [PATCH 73/75] docs(benchmark): document required environment manifest --- docs/doctoring/upload-ingestion-benchmark-methodology.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/doctoring/upload-ingestion-benchmark-methodology.md b/docs/doctoring/upload-ingestion-benchmark-methodology.md index 7393490a..283007a2 100644 --- a/docs/doctoring/upload-ingestion-benchmark-methodology.md +++ b/docs/doctoring/upload-ingestion-benchmark-methodology.md @@ -44,11 +44,16 @@ acceptance rules below. ```bash uv run python tools/benchmark_upload_ingestion.py \ --fixtures-dir /absolute/path/to/approved-pdf-fixtures \ + --environment-manifest /absolute/path/to/environment.json \ --output artifacts/upload-ingestion-result.json \ --repetitions 3 ``` -The default command runs the complete candidate and concurrency matrix. Filters +The environment manifest is a strict JSON object containing the execution image, +CPU model, memory bytes, storage device, filesystem, dependency-lock SHA-256, +exact 40-character commit SHA, worker count, and cache state +(`cold`, `warm`, or `disabled`). The harness rejects missing and unknown +fields. The default command runs the complete candidate and concurrency matrix. Filters are available for diagnosis, but a filtered run is not sufficient for a default selection. Execute the benchmark on an otherwise idle, dedicated Linux environment and record the container or host image, CPU model, memory, storage @@ -106,7 +111,7 @@ The checked-in JSON Schema is: docs/benchmarks/upload-ingestion-result.schema.json ``` -Accepted evidence must validate against schema version `1.1.0`. The report keeps +Accepted evidence must validate against schema version `1.2.0`. The report keeps request samples and raw cohort observations alongside aggregates, rejects unknown top-level and case fields, and records the exact resolved byte size even for the adaptive candidate. Generated reports are evidence artifacts, not source From 70d02297679f364acda47e895d9dbcd72e1a55b5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 20 Sep 2026 00:06:20 +0900 Subject: [PATCH 74/75] docs(readiness): state approved parser invariant --- src/newsdom_api/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/newsdom_api/main.py b/src/newsdom_api/main.py index 3afaf53e..ec6dd48b 100644 --- a/src/newsdom_api/main.py +++ b/src/newsdom_api/main.py @@ -201,7 +201,7 @@ def approved_parser_runtime_available() -> bool: def ready(request: Request) -> ReadinessResponse: - """Return readiness only when authentication and MinerU runtime are available.""" + """Return readiness only for valid authentication and an approved parser.""" settings = _runtime_settings(request) runtime_probe = request.app.state.runtime_readiness_probe From dd5ebcfac3f0163fb083741ae834d4f304cb4797 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 26 Sep 2026 20:09:04 +0900 Subject: [PATCH 75/75] test(docs): align public URL and loopback contracts --- tests/test_changelog.py | 8 ++++---- tests/test_manual_docs.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_changelog.py b/tests/test_changelog.py index 64eb9985..24d18095 100644 --- a/tests/test_changelog.py +++ b/tests/test_changelog.py @@ -27,11 +27,11 @@ def test_initial_release_entry_keeps_added_section_and_links(): in text ) assert ( - "[Unreleased]: https://github.com/Seongho-Bae/newsdom-api/compare/v0.2.0...HEAD" + "[Unreleased]: https://github.com/ContextualWisdomLab/newsdom-api/compare/v0.2.0...HEAD" in text ) assert ( - "[0.1.0]: https://github.com/Seongho-Bae/newsdom-api/releases/tag/v0.1.0" + "[0.1.0]: https://github.com/ContextualWisdomLab/newsdom-api/releases/tag/v0.1.0" in text ) @@ -42,10 +42,10 @@ def test_changelog_prepares_the_0_1_1_release_entry(): assert "### Added" in text assert "*.intoto.jsonl" in text assert ( - "[Unreleased]: https://github.com/Seongho-Bae/newsdom-api/compare/v0.2.0...HEAD" + "[Unreleased]: https://github.com/ContextualWisdomLab/newsdom-api/compare/v0.2.0...HEAD" in text ) assert ( - "[0.1.1]: https://github.com/Seongho-Bae/newsdom-api/compare/v0.1.0...v0.1.1" + "[0.1.1]: https://github.com/ContextualWisdomLab/newsdom-api/compare/v0.1.0...v0.1.1" in text ) diff --git a/tests/test_manual_docs.py b/tests/test_manual_docs.py index cdb63dec..af875650 100644 --- a/tests/test_manual_docs.py +++ b/tests/test_manual_docs.py @@ -69,7 +69,7 @@ def test_installation_doc_includes_manual_api_healthcheck_commands(): "uv run uvicorn", "--app-dir src", "newsdom_api.main:app", - "--host 0.0.0.0", + "--host 127.0.0.1", "--port 8000", "--reload", ]: