diff --git a/CHANGELOG.md b/CHANGELOG.md index 7221346c..1dab4e2a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,32 @@ in `CMakeLists.txt`) is derived from it. Suite-side only — no bngsim behavior changes. `parity_checks/rr_parity/ overrides.py` gains the entry and `ode_jobs.json` re-bakes the one job record. +- **Three `rr_parity` override keys that could never match are retired, and a + test now says so (#482 follow-up).** `MODEL0913003363:ode` (`TOL_OVERRIDES`), + `MODEL1006230083:ode` and `MODEL6963432821:ode` (both `NO_ORACLE_ADJUDICATED`) + name models the ODE manifest does not build: all three are `drop,no_model` in + the stage-1 corpus filter, their fetched files zero bytes (sha256 = the empty + string's), one of 305 corpus entries in that state. They have been stale since + the initial public release. + + A stale key is silent rot. It reads as a live disposition while covering + nothing, and it is the shape a typo'd or renamed `model_id` takes — an override + meant to reclassify a row that never did. The only detection was a + `print("WARN: ...")` in `build_ode_jobs.py` / `build_ssa_jobs.py`, which + nothing fails on and which cannot run at all in a checkout without + `$BIOMODELS_SEDML_DIR`. That is why three accumulated unnoticed. + + Each is retired as a comment rather than deleted, carrying its disposition + forward so it is not re-derived if the corpus ever fetches the model + successfully — the same treatment `MODEL2205030001:ode` got when GH #119 + retired it. `test_committed_overrides_have_no_stale_keys` then asserts the + committed overrides against the committed manifests for both regimes, which + needs neither the mirror nor the build. Verified to fail on an injected stale + key. + + Suite-side only; no bngsim behaviour changes, and no job's disposition moves — + none of these keys was reaching a job to begin with. + ## [0.15.0] - 2026-08-23 ### Added diff --git a/parity_checks/rr_parity/overrides.py b/parity_checks/rr_parity/overrides.py index 776042b3..24d8d7dd 100644 --- a/parity_checks/rr_parity/overrides.py +++ b/parity_checks/rr_parity/overrides.py @@ -348,16 +348,18 @@ "(both engines) zero real columns remain -> PASS. Stiff IVP." ), }, - "MODEL0913003363:ode": { - "rtol": 1e-10, - "atol": 1e-16, - "reason": ( - "Tolerance artifact (both engines converge when tightened). 'm' + 3 " - "other columns diverge sharply at the sweep tol (worst reldiv 0.951, " - "an early transient); at 1e-10/1e-16 (both engines) zero real columns " - "remain -> PASS. Ill-conditioned transient at the loose default." - ), - }, + # MODEL0913003363:ode — RETIRED 2026-08-25 as a stale key (GH #482 follow-up). + # The model is `drop,no_model` in the stage-1 corpus filter + # (benchmarks/suites/biomodels/manifest.csv): its fetched file is zero bytes + # (sha256 = the empty string's, e3b0c442...b855), one of 305 corpus entries in + # that state. build_ode_jobs.py therefore builds no job for it and the key + # could never match — `stale_keys` has flagged it since the initial public + # release. Recorded here rather than deleted outright, so the disposition is + # not re-derived if the corpus ever fetches the model successfully: + # tolerance artifact, both engines converge when tightened. 'm' + 3 other + # columns diverged sharply at the sweep tol (worst reldiv 0.951, an early + # transient); at rtol/atol 1e-10/1e-16 on BOTH engines zero real columns + # remained -> PASS. Ill-conditioned transient at the loose default. "BIOMD0000000827:ode": { "rtol": 1e-10, "atol": 1e-16, @@ -457,15 +459,13 @@ "below the significance floor). bngsim confirmed." ), }, - "MODEL1006230083:ode": { - "verdict": "confirm", - "issue": "GH #117", - "reason": ( - "RR CVODE CV_TOO_MUCH_WORK + COPASI failed. Pure rate-rule ODE system " - "on 5 parameters (no reactions). Independent scipy LSODA/BDF reproduce " - "to max_rel=0 (peak-rel ~1e-8). bngsim confirmed." - ), - }, + # MODEL1006230083:ode — RETIRED 2026-08-25 as a stale key (GH #482 follow-up), + # same cause as MODEL6963432821 below and MODEL0913003363 in TOL_OVERRIDES: + # `drop,no_model` in the stage-1 corpus filter, so no job is built and the key + # can never match. The GH #117 adjudication it carried, kept for the record: + # RR CVODE CV_TOO_MUCH_WORK + COPASI failed; pure rate-rule ODE system on 5 + # parameters (no reactions); independent scipy LSODA/BDF reproduce to + # max_rel=0 (peak-rel ~1e-8); verdict confirm — bngsim confirmed. "MODEL1112050001:ode": { "verdict": "confirm", "issue": "GH #117", @@ -495,15 +495,13 @@ "confirmed." ), }, - "MODEL6963432821:ode": { - "verdict": "confirm", - "issue": "GH #117", - "reason": ( - "RR CVODE CV_TOO_MUCH_WORK + COPASI failed. Pure rate-rule ODE system " - "on 7 parameters. Independent scipy LSODA/BDF reproduce to max_rel=0 " - "(peak-rel ~4e-9). bngsim confirmed." - ), - }, + # MODEL6963432821:ode — RETIRED 2026-08-25 as a stale key (GH #482 follow-up), + # same cause as MODEL1006230083 above: `drop,no_model` in the stage-1 corpus + # filter, so no job is built and the key can never match. The GH #117 + # adjudication it carried, kept for the record: RR CVODE CV_TOO_MUCH_WORK + + # COPASI failed; pure rate-rule ODE system on 7 parameters; independent scipy + # LSODA/BDF reproduce to max_rel=0 (peak-rel ~4e-9); verdict confirm — bngsim + # confirmed. "MODEL9811206584:ode": { "verdict": "confirm", "issue": "GH #117", diff --git a/parity_checks/tests/test_overrides.py b/parity_checks/tests/test_overrides.py index 5cfeea6a..c284223a 100644 --- a/parity_checks/tests/test_overrides.py +++ b/parity_checks/tests/test_overrides.py @@ -15,6 +15,10 @@ stale_keys * flags an authored key whose model is absent from the built set, and stays silent when every key is matched + * the COMMITTED overrides carry no stale key against the COMMITTED manifests + (the regression guard: build_*_jobs.py only `print`s a WARN, and cannot run + at all without $BIOMODELS_SEDML_DIR, so nothing else notices a key going + stale — three had accumulated undetected since the initial public release) _job_overrides (runner side) * reads a tol Override into a {rtol, atol} dict, and a known_artifact / @@ -25,10 +29,15 @@ from __future__ import annotations +from pathlib import Path + import overrides as ov -from _core import Job, Oracle, Override +import pytest +from _core import Job, Oracle, Override, read_manifest from rr_run import _job_overrides +RR_PARITY = Path(__file__).resolve().parents[1] / "rr_parity" + # --------------------------------------------------------------------------- # # overrides_for @@ -177,3 +186,29 @@ def test_job_overrides_reads_no_oracle_adjudicated(): tol, artifact, invalid_ref, adjudication = _job_overrides(job) assert tol is None and artifact is None and invalid_ref is None assert adjudication == ("confirm", "scipy BDF reproduces to max_rel=0", "GH #117") + + +# --------------------------------------------------------------------------- # +# The committed state +# --------------------------------------------------------------------------- # +@pytest.mark.parametrize("method,manifest", [("ode", "ode_jobs.json"), ("ssa", "ssa_jobs.json")]) +def test_committed_overrides_have_no_stale_keys(method, manifest): + """No authored override key may name a model the committed manifest doesn't + build. + + A stale key is silent rot: it reads as a live disposition while covering + nothing, and it is the shape a typo'd or renamed model_id takes — an + override that was meant to reclassify a row and never did. The only existing + detection is a ``print("WARN: ...")`` in build_ode_jobs.py / build_ssa_jobs.py, + which nothing fails on and which cannot run at all in a checkout without + ``$BIOMODELS_SEDML_DIR``; three ODE keys had accumulated undetected since the + initial public release (retired in the GH #482 follow-up, with their + dispositions kept as comments in overrides.py). This asserts the committed + overrides against the committed manifests, which needs neither. + + Retiring a key is the fix when the model genuinely left the corpus; correcting + it is the fix when the model_id is wrong. Both are better than a warning + nobody reads. + """ + _, jobs = read_manifest(RR_PARITY / manifest) + assert ov.stale_keys({j.model_id for j in jobs}, method) == []