diff --git a/CHANGELOG.md b/CHANGELOG.md index 2398ea5c..02d97214 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - OpenAPI 제목/설명, README, `ArticleNode.headline` 문서를 일반 문서용 (section heading) 표현으로 재구성하여 특정 언어/신문 가정을 소비자에게 노출하지 않도록 함. 응답 스키마 필드는 하위 호환을 위해 변경하지 않음. ### Added +- [CLI] 파싱된 NewsDOM JSON 데이터를 JSON Lines(JSONL) 포맷으로 추출하여 기사 단위 데이터셋 처리를 용이하게 하는 `tools/export_jsonl.py` 도구를 추가했습니다. - [CLI] 단일 NewsDOM JSON 파일을 페이지 단위로 분리하는 `tools/split_dom.py` 도구를 추가했습니다. - [CLI] NewsDOM JSON 파일의 모든 텍스트 내용을 마스킹하여 익명화하는 `tools/anonymize_dom.py` 도구를 추가했습니다. - `/parse`에 기본 필수 bearer 인증 경계를 추가했습니다. `NEWSDOM_AUTH_MODE=required`, `NEWSDOM_RUNTIME_PROFILE=production`, `NEWSDOM_API_TOKEN`을 명시해야 하며, 인증 비활성화는 격리된 development 프로필에서만 허용됩니다. `/health`는 liveness 전용으로 미인증 상태를 유지하고 `/ready`가 인증 설정과 MinerU 가용성을 함께 검증합니다. diff --git a/tests/test_tools_export_jsonl.py b/tests/test_tools_export_jsonl.py new file mode 100644 index 00000000..7658e6d1 --- /dev/null +++ b/tests/test_tools_export_jsonl.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from tools.export_jsonl import export_jsonl, main + +VALID_JSON_DATA = { + "document_id": "test_doc", + "pages": [ + { + "page_number": 1, + "articles": [ + { + "article_id": "art_1", + "headline": "Test Headline 1", + "body_blocks": ["Block 1", "Block 2"], + }, + { + "article_id": "art_2", + "headline": "Test Headline 2", + "body_blocks": [], + }, + ], + }, + "not_a_dict_page", + { + "page_number": 2, + "articles": [ + "not_a_dict_article", + { + "article_id": "art_3", + "headline": "Test Headline 3", + "body_blocks": ["Block 3"], + }, + ], + }, + ], +} + + +def test_export_jsonl_success(tmp_path: Path) -> None: + input_file = tmp_path / "input.json" + input_file.write_text(json.dumps(VALID_JSON_DATA), encoding="utf-8") + output_file = tmp_path / "output.jsonl" + + export_jsonl(input_file, output_file) + + assert output_file.exists() + + with output_file.open("r", encoding="utf-8") as f: + lines = f.readlines() + + assert len(lines) == 3 + record0 = json.loads(lines[0]) + assert record0["document_id"] == "test_doc" + assert record0["page_number"] == 1 + assert record0["article_id"] == "art_1" + assert record0["headline"] == "Test Headline 1" + assert record0["body_blocks"] == ["Block 1", "Block 2"] + + record1 = json.loads(lines[1]) + assert record1["article_id"] == "art_2" + assert record1["body_blocks"] == [] + + record2 = json.loads(lines[2]) + assert record2["page_number"] == 2 + assert record2["article_id"] == "art_3" + assert record2["body_blocks"] == ["Block 3"] + + +def test_export_jsonl_invalid_file(tmp_path: Path) -> None: + output_file = tmp_path / "output.jsonl" + + non_existent = tmp_path / "not_exist.json" + with pytest.raises(FileNotFoundError, match="File not found"): + export_jsonl(non_existent, output_file) + + not_json = tmp_path / "input.txt" + not_json.write_text("plain text", encoding="utf-8") + with pytest.raises(ValueError, match="must be a .json file"): + export_jsonl(not_json, output_file) + + invalid_json = tmp_path / "invalid.json" + invalid_json.write_text("{invalid_json:", encoding="utf-8") + with pytest.raises(ValueError, match="Invalid JSON file"): + export_jsonl(invalid_json, output_file) + + +def test_export_jsonl_cli_success(tmp_path: Path, capsys: pytest.CaptureFixture) -> None: + input_file = tmp_path / "input.json" + input_file.write_text(json.dumps(VALID_JSON_DATA), encoding="utf-8") + output_file = tmp_path / "output.jsonl" + + main([str(input_file), str(output_file)]) + + assert output_file.exists() + captured = capsys.readouterr() + assert "JSONL successfully written" in captured.out + + +def test_export_jsonl_cli_invalid_file( + tmp_path: Path, capsys: pytest.CaptureFixture +) -> None: + not_json = tmp_path / "input.txt" + not_json.write_text("plain text", encoding="utf-8") + output_file = tmp_path / "output.jsonl" + + with pytest.raises(SystemExit) as exc_info: + main([str(not_json), str(output_file)]) + + assert exc_info.value.code == 1 + captured = capsys.readouterr() + assert "Error exporting JSONL:" in captured.err diff --git a/tools/export_jsonl.py b/tools/export_jsonl.py new file mode 100644 index 00000000..3d5dc075 --- /dev/null +++ b/tools/export_jsonl.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +def export_jsonl(json_path: Path, output_path: Path) -> None: + """Export NewsDOM JSON to a JSONL file containing article objects.""" + if not json_path.is_file(): + raise FileNotFoundError(f"File not found or is not a file: {json_path}") + if json_path.suffix.lower() != ".json": + raise ValueError("Input file must be a .json file.") + + try: + data = json.loads(json_path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise ValueError(f"Invalid JSON file: {exc}") from exc + + pages = data.get("pages", []) + document_id = data.get("document_id", "Unknown Document") + + with output_path.open("w", encoding="utf-8") as f: + for page in pages: + if not isinstance(page, dict): + continue + page_number = page.get("page_number", "Unknown") + articles = page.get("articles", []) + for article in articles: + if not isinstance(article, dict): + continue + record = { + "document_id": document_id, + "page_number": page_number, + "article_id": article.get("article_id", "Unknown Article ID"), + "headline": article.get("headline", ""), + "body_blocks": article.get("body_blocks", []), + } + f.write(json.dumps(record, ensure_ascii=False) + "\n") + + +def main(argv: list[str] | None = None) -> None: + """Run the JSON-to-JSONL export CLI.""" + parser = argparse.ArgumentParser(description="Export a NewsDOM JSON file to JSONL.") + parser.add_argument("input", type=Path, help="Path to the input JSON file.") + parser.add_argument("output", type=Path, help="Path to write the JSONL output file.") + + args = parser.parse_args(argv) + + try: + export_jsonl(args.input, args.output) + print(f"JSONL successfully written to {args.output}") + except Exception as exc: + print(f"Error exporting JSONL: {exc}", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": # pragma: no cover + main() diff --git a/uv.lock b/uv.lock index a0d133b8..1ca380d4 100644 --- a/uv.lock +++ b/uv.lock @@ -303,7 +303,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -929,14 +929,14 @@ wheels = [ [[package]] name = "pypdf" -version = "6.15.0" +version = "6.16.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/17/17/ee75a92718ec7212de831e71454d702225aa5e474a805cce169806044453/pypdf-6.15.0.tar.gz", hash = "sha256:d39c4d955a76409284a905e2d65b40076d77ab76129e0faaeeb6612403ecfc79", size = 6993794, upload-time = "2026-08-06T13:06:49.929Z" } +sdist = { url = "https://files.pythonhosted.org/packages/44/66/54212e75406afd9f3e933d0dda23072f6aecc55c5a273077dc2e0b028b23/pypdf-6.16.2.tar.gz", hash = "sha256:595647f6191de6f402cfde1d0c455d6cbccbd509aac32b34783009c032de5d6e", size = 7008996, upload-time = "2026-08-23T13:50:07.135Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/af/72/ce3067ac31e214a66388159f8462ddb8c13dd00170f24d555a1f1ae8ee91/pypdf-6.15.0-py3-none-any.whl", hash = "sha256:14e001d6504822cb1ca9c7ed9a69bccb320f59b320730f55af804361abe4d5ee", size = 378123, upload-time = "2026-08-06T13:06:47.709Z" }, + { url = "https://files.pythonhosted.org/packages/13/f1/a2da3b55acd4ab737bf728c97edaaed5ec1d3c1236acb639dcdfa97e42c7/pypdf-6.16.2-py3-none-any.whl", hash = "sha256:c8b09a59399062fb45a1b8156c18a787a10a3dae03ac9674397a226712c94604", size = 385060, upload-time = "2026-08-23T13:50:05.349Z" }, ] [[package]]