feat: JSONL 추출을 위한 export_jsonl.py 도구 추가

CHANGELOG.md

@@ -20,6 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - OpenAPI 제목/설명, README, `ArticleNode.headline` 문서를 일반 문서용 (section heading) 표현으로 재구성하여 특정 언어/신문 가정을 소비자에게 노출하지 않도록 함. 응답 스키마 필드는 하위 호환을 위해 변경하지 않음.
 
 ### Added
+- [CLI] 파싱된 NewsDOM JSON 데이터를 JSON Lines(JSONL) 포맷으로 추출하여 기사 단위 데이터셋 처리를 용이하게 하는 `tools/export_jsonl.py` 도구를 추가했습니다.
 - [CLI] 단일 NewsDOM JSON 파일을 페이지 단위로 분리하는 `tools/split_dom.py` 도구를 추가했습니다.
 - [CLI] NewsDOM JSON 파일의 모든 텍스트 내용을 마스킹하여 익명화하는 `tools/anonymize_dom.py` 도구를 추가했습니다.
 - `/parse`에 기본 필수 bearer 인증 경계를 추가했습니다. `NEWSDOM_AUTH_MODE=required`, `NEWSDOM_RUNTIME_PROFILE=production`, `NEWSDOM_API_TOKEN`을 명시해야 하며, 인증 비활성화는 격리된 development 프로필에서만 허용됩니다. `/health`는 liveness 전용으로 미인증 상태를 유지하고 `/ready`가 인증 설정과 MinerU 가용성을 함께 검증합니다.

tests/test_tools_export_jsonl.py

@@ -0,0 +1,116 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+from tools.export_jsonl import export_jsonl, main
+
+VALID_JSON_DATA = {
+    "document_id": "test_doc",
+    "pages": [
+        {
+            "page_number": 1,
+            "articles": [
+                {
+                    "article_id": "art_1",
+                    "headline": "Test Headline 1",
+                    "body_blocks": ["Block 1", "Block 2"],
+                },
+                {
+                    "article_id": "art_2",
+                    "headline": "Test Headline 2",
+                    "body_blocks": [],
+                },
+            ],
+        },
+        "not_a_dict_page",
+        {
+            "page_number": 2,
+            "articles": [
+                "not_a_dict_article",
+                {
+                    "article_id": "art_3",
+                    "headline": "Test Headline 3",
+                    "body_blocks": ["Block 3"],
+                },
+            ],
+        },
+    ],
+}
+
+
+def test_export_jsonl_success(tmp_path: Path) -> None:
+    input_file = tmp_path / "input.json"
+    input_file.write_text(json.dumps(VALID_JSON_DATA), encoding="utf-8")
+    output_file = tmp_path / "output.jsonl"
+
+    export_jsonl(input_file, output_file)
+
+    assert output_file.exists()
+
+    with output_file.open("r", encoding="utf-8") as f:
+        lines = f.readlines()
+
+        assert len(lines) == 3
+        record0 = json.loads(lines[0])
+        assert record0["document_id"] == "test_doc"
+        assert record0["page_number"] == 1
+        assert record0["article_id"] == "art_1"
+        assert record0["headline"] == "Test Headline 1"
+        assert record0["body_blocks"] == ["Block 1", "Block 2"]
+
+        record1 = json.loads(lines[1])
+        assert record1["article_id"] == "art_2"
+        assert record1["body_blocks"] == []
+
+        record2 = json.loads(lines[2])
+        assert record2["page_number"] == 2
+        assert record2["article_id"] == "art_3"
+        assert record2["body_blocks"] == ["Block 3"]
+
+
+def test_export_jsonl_invalid_file(tmp_path: Path) -> None:
+    output_file = tmp_path / "output.jsonl"
+
+    non_existent = tmp_path / "not_exist.json"
+    with pytest.raises(FileNotFoundError, match="File not found"):
+        export_jsonl(non_existent, output_file)
+
+    not_json = tmp_path / "input.txt"
+    not_json.write_text("plain text", encoding="utf-8")
+    with pytest.raises(ValueError, match="must be a .json file"):
+        export_jsonl(not_json, output_file)
+
+    invalid_json = tmp_path / "invalid.json"
+    invalid_json.write_text("{invalid_json:", encoding="utf-8")
+    with pytest.raises(ValueError, match="Invalid JSON file"):
+        export_jsonl(invalid_json, output_file)
+
+
+def test_export_jsonl_cli_success(tmp_path: Path, capsys: pytest.CaptureFixture) -> None:
+    input_file = tmp_path / "input.json"
+    input_file.write_text(json.dumps(VALID_JSON_DATA), encoding="utf-8")
+    output_file = tmp_path / "output.jsonl"
+
+    main([str(input_file), str(output_file)])
+
+    assert output_file.exists()
+    captured = capsys.readouterr()
+    assert "JSONL successfully written" in captured.out
+
+
+def test_export_jsonl_cli_invalid_file(
+    tmp_path: Path, capsys: pytest.CaptureFixture
+) -> None:
+    not_json = tmp_path / "input.txt"
+    not_json.write_text("plain text", encoding="utf-8")
+    output_file = tmp_path / "output.jsonl"
+
+    with pytest.raises(SystemExit) as exc_info:
+        main([str(not_json), str(output_file)])
+
+    assert exc_info.value.code == 1
+    captured = capsys.readouterr()
+    assert "Error exporting JSONL:" in captured.err

tools/export_jsonl.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+
+def export_jsonl(json_path: Path, output_path: Path) -> None:
+    """Export NewsDOM JSON to a JSONL file containing article objects."""
+    if not json_path.is_file():
+        raise FileNotFoundError(f"File not found or is not a file: {json_path}")
+    if json_path.suffix.lower() != ".json":
+        raise ValueError("Input file must be a .json file.")
+
+    try:
+        data = json.loads(json_path.read_text(encoding="utf-8"))
+    except json.JSONDecodeError as exc:
+        raise ValueError(f"Invalid JSON file: {exc}") from exc
+
+    pages = data.get("pages", [])
+    document_id = data.get("document_id", "Unknown Document")
+
+    with output_path.open("w", encoding="utf-8") as f:
+        for page in pages:
+            if not isinstance(page, dict):
+                continue
+            page_number = page.get("page_number", "Unknown")
+            articles = page.get("articles", [])
+            for article in articles:
+                if not isinstance(article, dict):
+                    continue
+                record = {
+                    "document_id": document_id,
+                    "page_number": page_number,
+                    "article_id": article.get("article_id", "Unknown Article ID"),
+                    "headline": article.get("headline", ""),
+                    "body_blocks": article.get("body_blocks", []),
+                }
+                f.write(json.dumps(record, ensure_ascii=False) + "\n")
+
+
+def main(argv: list[str] | None = None) -> None:
+    """Run the JSON-to-JSONL export CLI."""
+    parser = argparse.ArgumentParser(description="Export a NewsDOM JSON file to JSONL.")
+    parser.add_argument("input", type=Path, help="Path to the input JSON file.")
+    parser.add_argument("output", type=Path, help="Path to write the JSONL output file.")
+
+    args = parser.parse_args(argv)
+
+    try:
+        export_jsonl(args.input, args.output)
+        print(f"JSONL successfully written to {args.output}")
+    except Exception as exc:
+        print(f"Error exporting JSONL: {exc}", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":  # pragma: no cover
+    main()