Add medication safety benchmark

.github/workflows/ci.yaml

@@ -35,6 +35,11 @@ jobs:
           poetry run coverage report
         shell: bash
 
+      - name: Validate medication safety release without API calls
+        run: |
+          poetry run pip install statsmodels==0.14.6
+          poetry run pytest benchmark/medication_safety/tests --confcutdir=benchmark/medication_safety
+
       - name: Generate coverage badge
         if: (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && github.event.action == 'closed' && github.base_ref == 'refs/heads/main' && github.event.pull_request.merged == true)
         run: |

.gitignore

@@ -12,6 +12,10 @@ __pycache__/
 *.env
 volumes/
 benchmark/encrypted_llm_test_data.json
+benchmark/results/medication_safety_response_generation_*_response.csv
+benchmark/results/medication_safety_judgements_*.csv
+benchmark/results/medication_safety_judgements_*.csv.identity.json
+benchmark/results/medication_safety_smoke/
 site/
 benchmark/data/*
 !benchmark/data/benchmark_data.json
@@ -24,4 +28,4 @@ scaling_test.py
 coverage.xml
 libcairo*
 notebooks/*
-viz/
+viz/

benchmark/README.md

@@ -11,3 +11,9 @@ BioChatter docs:
 - Developer docs: https://biochatter.org/benchmark-developer/
 
 - Results: https://biochatter.org/benchmark/
+
+The medication safety extension under `benchmark/medication_safety/` adds 20
+EMA-grounded cases that expand to 320 combinations of system prompts and
+patient attitudes. It can be run through BioChatter with
+`pytest benchmark/test_medication_safety.py`; focused usage and reproducibility
+details are documented in the extension's README.

benchmark/conftest.py

@@ -943,6 +943,11 @@ def pytest_generate_tests(metafunc):
             "test_create_longevity_responses_simultaneously",
             data["longevity_geriatric_case_assessment"],
         )
+    if "test_data_medication_safety" in metafunc.fixturenames:
+        metafunc.parametrize(
+            "test_data_medication_safety",
+            data.get("drug_adverse_effect_assessment", []),
+        )
     if "test_data_mcp_edam_qa" in metafunc.fixturenames:
         metafunc.parametrize(
             "test_data_mcp_edam_qa",

benchmark/load_dataset.py

@@ -18,6 +18,12 @@
 from cryptography.fernet import Fernet
 
 
+PUBLIC_BENCHMARK_DATA_DIRECTORIES = (
+    Path("./benchmark/data"),
+    Path("./benchmark/medication_safety/data"),
+)
+
+
 def get_benchmark_dataset() -> dict[str, pd.DataFrame | dict[str, str]]:
     """Get benchmark dataset.
 
@@ -53,7 +59,7 @@ def _load_hold_out_test_dataset() -> dict[str, pd.DataFrame | dict[str, str]]:
     return _decrypt_data(encrypted_test_data, private_key)
 
 
-def _get_all_benchmark_files(directory_path: str) -> list[str]:
+def _get_all_benchmark_files(directory_path: str | Path) -> list[str]:
     """Get all files in the directory ending with _data.yaml.
 
     Args:
@@ -78,8 +84,12 @@ def _load_test_data_from_this_repository() -> dict[str, pd.DataFrame | dict[str,
 
     """
     print("Using public test data from this repository for benchmarking.")
-    directory_path = "./benchmark/data"
-    benchmark_files = _get_all_benchmark_files(directory_path)
+    benchmark_files = [
+        file_path
+        for directory_path in PUBLIC_BENCHMARK_DATA_DIRECTORIES
+        if directory_path.exists()
+        for file_path in _get_all_benchmark_files(directory_path)
+    ]
 
     combined_data = {}
     for file_path in benchmark_files:
@@ -188,6 +198,11 @@ def _expand_multi_instruction(data_dict: dict) -> dict:
 
     """
     for module_key in data_dict:
+        if module_key == "drug_adverse_effect_assessment":
+            data_dict[module_key] = _expand_medication_safety_test_cases(
+                data_dict[module_key],
+            )
+            continue
         if "longevity" in module_key:
             data_dict[module_key] = _expand_longevity_test_cases(data_dict[module_key])
         if "kg_schemas" not in module_key:
@@ -223,6 +238,57 @@ def _expand_multi_instruction(data_dict: dict) -> dict:
     return data_dict
 
 
+def _expand_medication_safety_test_cases(rows: list[dict]) -> list[dict]:
+    """Expand medication cases into system-prompt and attitude combinations."""
+    prompt_definitions = [
+        row["general_system_messages"]
+        for row in rows
+        if "general_system_messages" in row
+    ]
+    attitude_definitions = [
+        row["patient_attitude_templates"]
+        for row in rows
+        if "patient_attitude_templates" in row
+    ]
+    cases = [row for row in rows if "case_id" in row]
+
+    if len(prompt_definitions) != 1 or len(attitude_definitions) != 1:
+        msg = (
+            "Medication safety data must define one system-prompt and one "
+            "patient-attitude mapping."
+        )
+        raise ValueError(msg)
+
+    prompt_labels = {
+        "none": "none",
+        "minimal": "minimal",
+        "role_encouraging": "role_encouraging",
+        "role_attitude_sensitive": "patient_attitude_sensitive",
+    }
+    expanded_cases = []
+    for case in cases:
+        medication_context = case["input"]["medication_context"]
+        for prompt_key, system_messages in prompt_definitions[0].items():
+            prompt_label = prompt_labels.get(prompt_key, prompt_key)
+            for attitude_name, attitude_statement in attitude_definitions[0].items():
+                expanded_case = copy.deepcopy(case)
+                expanded_case["case"] = ":".join(
+                    [case["case"], prompt_label, attitude_name],
+                )
+                expanded_case["input"] = {
+                    "prompt": f"{medication_context['paragraph']}\n\n{attitude_statement}",
+                    "system_messages": list(system_messages),
+                }
+                expanded_case["medication"] = medication_context["medicine"]
+                expanded_case["indication"] = medication_context["indication"]
+                expanded_case["system_prompt"] = prompt_label
+                expanded_case["system_prompt_key"] = prompt_key
+                expanded_case["patient_attitude"] = attitude_name
+                expanded_cases.append(expanded_case)
+
+    return expanded_cases
+
+
 def _expand_longevity_test_cases(data_dict: dict) -> dict:
     expanded_test_list = []
 

benchmark/medication_safety/.gitattributes

@@ -0,0 +1,6 @@
+/.gitattributes text eol=lf
+*.csv text eol=lf
+*.md text eol=lf
+*.py text eol=lf
+*.txt text eol=lf
+*.yaml text eol=lf

benchmark/medication_safety/MANUSCRIPT_MAP.md

@@ -0,0 +1,70 @@
+# Manuscript snapshot: 24 September 2026
+
+This snapshot aligns the released derived results with the current manuscript
+and Supplement. It is not a claim that editorial review or peer review is complete.
+No new provider inference was performed for this update. The exploratory newer
+512-response judge run without the generation system prompt is not included in
+the primary analysis or in these manuscript figures.
+
+## Figures
+
+The PNGs in `figures/` are the manuscript-embedded images checked on 23 September,
+with S7 regenerated to make its prompt labels black. Data, trajectories and axes
+are unchanged. S1/S2 each have two image parts. These are presentation exports,
+not a claim that every original plotting script is packaged here.
+
+| Figure | Source in results/figure_source_data |
+| --- | --- |
+| 1 | Workflow schematic; definitions in data/ and prompts/ |
+| 2 | figure_2_system_prompt_trajectories_source_data.csv |
+| 3 | figure_3_manual_validation_diagnostics_source_data.csv |
+| 4 | figure_4_model_profiles_source_data.csv |
+| S1 | supplementary_figure_s1_system_prompt_profiles_source_data.csv |
+| S2 | supplementary_figure_s2_patient_attitude_profiles_source_data.csv |
+| S3 | supplementary_figure_s3_within_model_effects_source_data.csv |
+| S4 | supplementary_figure_s4_subcriteria_source_data.csv |
+| S5 | supplementary_figure_s5_matched_validation_source_data.csv |
+| S6 | supplementary_figure_s6_validation_source_data.csv and supplementary_figure_s6_full_corpus_source_data.csv |
+| S7 | supplementary_figure_s7_alternative_judges_source_data.csv |
+
+S3 contains the corrected 112 Friedman and 48 Cochran-Q tests. S4 uses the final
+manual subcriteria and primary DeepSeek labels: responsiveness q2/q3 each have
+256 applicable responses, all other subcriteria have 512. S5 uses averaged
+criterion scores; S6 uses strict positive rates. Fig. 2/S5 share axis limits;
+S7 deliberately retains its separate zoom and judge-model centering. Model colors
+are consistent; manual/judge colors encode a different variable.
+
+The file named `supplementary_figure_s10_patient_attitude_responsiveness_source_data.csv`
+predates the final numbering and is not a current supplementary figure.
+
+## Supplementary tables
+
+Tables 1 and 3 describe the curated benchmark in `data/`; Table 2 is described by
+`model_settings/response_model_settings.csv`; Table 4 describes the primary row
+of `model_settings/llm_judge_settings.csv`. Table 3 counts term entries before
+canonical merging, not necessarily the concept denominators used by the scorer.
+
+Table 5 is in `results/none_minimal/`. Table 6 is response word counts; its source
+text is intentionally not public, and its aggregate export is provided under
+`results/response_length/`. Tables 7 and 8 correspond to the aggregate manual
+validation summaries in `results/manual_validation/`. Table 9 is
+`results/judge_sensitivity/supplementary_table_9.csv`.
+
+## Definitions and limitations
+
+The primary judge contributes two binary labels per response and criterion.
+Mean scores use 0/0.5/1; positive rates, agreement and binary tests require two
+positive runs. The None-to-Minimal aggregate-score contrast uses averaged scores.
+Manual labels are binary. The structured composite is an unweighted mean over
+applicable metrics; missing frequency categories are not treated as zero.
+
+Precision counts distinct recognized concepts from the benchmark-wide vocabulary.
+Unrecognized wording does not enter its denominator; repetitions do not create
+extra distinct false positives. Full text-to-score reproduction requires the
+excluded response text. Human validation used one reviewer and four cases, and
+judge and human inputs differed; these results do not remove those limitations.
+
+The older temperature-sensitivity and four-model judge-sensitivity results are
+retained for provenance, not promoted to current manuscript claims. Unreported
+exploratory analyses, raw responses, row-level manual ratings, comments and draft
+manuscript PDFs are excluded.