From a0a8f64da366bb76df374ff1d9ebb4764e5ce8f6 Mon Sep 17 00:00:00 2001 From: Anwesh Nayak Date: Mon, 9 Nov 2026 15:17:42 +0000 Subject: [PATCH 1/3] test(triage): add judge calibration and baseline scorecards --- .../core/telemetry/evaluation/CALIBRATION.md | 136 ++++++++ tests/core/telemetry/evaluation/README.md | 5 + .../telemetry/evaluation/calibration.json | 51 +++ .../core/telemetry/evaluation/calibration.py | 298 ++++++++++++++++++ tests/core/telemetry/evaluation/reporting.py | 212 +++++++++++++ tests/core/telemetry/test_calibration.py | 190 +++++++++++ 6 files changed, 892 insertions(+) create mode 100644 tests/core/telemetry/evaluation/CALIBRATION.md create mode 100644 tests/core/telemetry/evaluation/calibration.json create mode 100644 tests/core/telemetry/evaluation/calibration.py create mode 100644 tests/core/telemetry/evaluation/reporting.py create mode 100644 tests/core/telemetry/test_calibration.py diff --git a/tests/core/telemetry/evaluation/CALIBRATION.md b/tests/core/telemetry/evaluation/CALIBRATION.md new file mode 100644 index 0000000000..92b60c6671 --- /dev/null +++ b/tests/core/telemetry/evaluation/CALIBRATION.md @@ -0,0 +1,136 @@ +# Calibrating the investigator baseline (stacked draft) + +This builds on the baseline tooling in #6994. It adds review material and offline +comparison/reporting; it does not establish human judgments or live model scores. +All source cases and authored reports still await SRE review. + +## Prepare a review pack + +From the repository root, using the normal OpenSRE environment: + +```bash +uv run python -m tests.core.telemetry.evaluation.calibration prepare \ + --seed 42 --output work/triage-calibration-review +``` + +The required output directory must be new. Optional `--cases` and `--examples` +default to the included `cases.json` and `calibration.json`; `--seed` defaults +to 42. There are nine authored reports across three development cases: invalid +token, token expiry supported by a validator log, and absent evidence. Holdout +cases cannot be used for calibration. + +The directory contains: + +- `runs.jsonl`: authored reports in the baseline scorer's input format, with + `purpose=judge_calibration` and explicit non-investigator origin. Preparation + makes no model calls. The `completed` status means fixture assembly completed, + not that a model generated a successful diagnosis. +- `review.json`: redacted evidence, proposed case expectations and candidate + reports, with neutral stable IDs and shuffled ordering. Author notes and + suggested quality labels are omitted. All ratings start pending/null. The + source examples remain visible in Git; this is a review aid, not a claim of + experimental blinding. + +Have an SRE review the case expectations first. In the worksheet, record their +identity in `reviewer`, set each case `decision` to `accepted` or `revise`, and +add notes. A case marked `revise` is excluded from agreement comparisons; correct +its source expectations and prepare a new pack before accepting it. Changing +evidence, proposed expectations, report text or IDs inside an existing pack is +rejected during comparison. + +For each report, fill both integer ratings (0–3), a rationale, and set +`review_status` to `reviewed`. Leave unfinished entries pending; retain them in +the file. Proposed anchors to discuss with the SRE: + +| Rating | Evidence support | Diagnostic usefulness | +|---|---|---| +| 0 | Materially false or unsupported | Wrong or misleading | +| 1 | Mixed support and unsupported claims | Mostly repeats symptoms | +| 2 | Mostly supported, minor gaps | Useful finding with some omissions | +| 3 | Supported with appropriate limitations | Strongest justified finding and targeted next check | + +Correct uncertainty can earn full credit on missing-evidence cases. Unnecessary +abstention loses usefulness credit when decisive evidence exists. A matching +symptom does not justify an invented deployment or underlying cause. These +anchors and example reports are proposals pending SRE validation. + +## Judge the authored reports + +Use the existing isolated, locked DeepEval project with a configured judge model: + +```bash +DEEPEVAL_TELEMETRY_OPT_OUT=1 uv run --locked \ + --project tests/core/telemetry/evaluation/judge \ + python tests/core/telemetry/evaluation/judge/score.py \ + --runs work/triage-calibration-review/runs.jsonl \ + --output work/triage-calibration-review/scores.jsonl \ + --model gpt-4.1 +``` + +The model is an example selector. This is the only live step in this calibration +workflow and consumes judge quota. It receives redacted incident evidence, the +proposed case rubric, and each report. It receives no human ratings or author +notes. Do not show judge results to the reviewer until their ratings are saved. +To assess judge variation, rescore the same pack to a new output file. + +## Compare against human ratings + +```bash +uv run python -m tests.core.telemetry.evaluation.calibration compare \ + --runs work/triage-calibration-review/runs.jsonl \ + --scores work/triage-calibration-review/scores.jsonl \ + --review work/triage-calibration-review/review.json \ + --output work/triage-calibration-review/comparison.json +``` + +This offline command binds both judgments to the same immutable input snapshot. +Only attributed, completed ratings for accepted case expectations count. It +compares report preferences **within each case and metric**; the human ordinal +scale is not treated as equivalent to DeepEval's normalized score scale. + +Output includes comparable pairs, agreements, human ties, judge ties, missing +judge pairs and individual disagreements. Human ties are excluded from the +agreement denominator; judge ties disagree with a strict human preference. +With no eligible comparisons, agreement is `null`, never perfect. These small, +correlated example pairs do not establish statistical confidence or investigator +accuracy. The command does not mark source fixtures as SRE-reviewed. + +## Produce the real baseline scorecard + +Run the [baseline generation and scoring commands](README.md) on investigator +outputs, then: + +```bash +uv run python -m tests.core.telemetry.evaluation.reporting \ + --runs work/triage-baseline.jsonl \ + --scores work/triage-baseline-scores.jsonl \ + --output work/triage-baseline-scorecard.json +``` + +The scorecard rejects authored calibration packs. It validates that score results +match their metric checkpoints and exact generation snapshot, then groups +results by failure class. It exposes generation failures, partial results, +missing attempts, unfinished judging, per-case metric variation and durations. +An interrupted score file retains completed metric checkpoints; absent metrics +remain missing rather than becoming zero or perfect scores. Stored summary +statistics are ignored and recomputed. Fixture review counts refer to unique +recorded cases, not repeat attempts. + +All output files must be new; generated material stays under ignored `work/`. +Readers accept an incomplete final JSONL write while flagging it, and reject +damaged complete rows, duplicate score events, changed generation facts and +invalid metric values. + +## Baseline v0 readiness + +Before treating the baseline as useful measurement, SRE-review all incident +expectations, inspect human/judge disagreements, and collect repeated live +investigator results with a fixed model configuration and rubric. Record the +source hashes and missing/failing attempts. Unsupported-claim counts, useful +diagnosis counts and appropriate-abstention counts still require claim-level +SRE review; a semantic score is not a verified count of those outcomes. + +This draft supplies no live score report, human rating or calibrated threshold. +The previous live investigator smoke was blocked by the environment proxy; +successful investigator and judge access remain prerequisites for collecting +real results. Ordinary pytest and CI remain offline. diff --git a/tests/core/telemetry/evaluation/README.md b/tests/core/telemetry/evaluation/README.md index e0f9d835e1..0cde035b9f 100644 --- a/tests/core/telemetry/evaluation/README.md +++ b/tests/core/telemetry/evaluation/README.md @@ -154,3 +154,8 @@ then compare structured-report v1 on identical evidence/models. Add the scoped tool-calling investigator only after those comparisons. Improvements must be shown per case alongside completion/abstention behavior; an agent that always says "unknown" must not win by avoiding every diagnosis. + +The stacked [calibration workflow](CALIBRATION.md) prepares neutral review +material, compares attributed human ratings with judge scores, and produces +failure-class scorecards from saved investigator runs. Its authored examples +are explicitly separate from live investigator results. diff --git a/tests/core/telemetry/evaluation/calibration.json b/tests/core/telemetry/evaluation/calibration.json new file mode 100644 index 0000000000..dcccefa6e8 --- /dev/null +++ b/tests/core/telemetry/evaluation/calibration.json @@ -0,0 +1,51 @@ +{ + "version": 1, + "review_status": "pending_sre_review", + "examples": [ + { + "case_id": "auth_rejection", + "report": "Observed: 30/30 failed requests in the configured service/window.\nThe selected trace trace-001/span-001 reports Invalid token. Token validation rejection is indicated for that request. Why the token is invalid and how many other failures share this mechanism remain unknown. Next check: compare affected callers and inspect token validation configuration.", + "author_note": "Proposed useful supported report; SRE assessment pending." + }, + { + "case_id": "auth_rejection", + "report": "Observed: 30/30 failed requests in the configured service/window.\nThere is an incident. Cause unknown. Next check: investigate.", + "author_note": "Proposed underhelpful abstention; SRE assessment pending." + }, + { + "case_id": "auth_rejection", + "report": "Observed: 30/30 failed requests in the configured service/window.\nThe latest deployment rotated the provider credential incorrectly. All 30 requests failed because of that rotation. Roll back the deployment to fix the incident.", + "author_note": "Proposed unsupported causal narrative; SRE assessment pending." + }, + { + "case_id": "auth_expiry_supported", + "report": "Observed: 30/30 failed requests in the configured service/window.\nTrace trace-001/span-001 reports Invalid token. Its validator log says request_timestamp=170 and token_expiry_timestamp=140 and explicitly rejects the expired token. Expiry explains that selected rejection. Why refresh failed and whether other requests share this cause remain unknown. Next check: inspect token refresh and other failed requests.", + "author_note": "Proposed useful finding from decisive evidence; SRE assessment pending." + }, + { + "case_id": "auth_expiry_supported", + "report": "Observed: 30/30 failed requests in the configured service/window.\nCause unknown. There are too few logs to say anything useful. Next check: inspect the logs.", + "author_note": "Proposed unnecessary abstention despite evidence; SRE assessment pending." + }, + { + "case_id": "auth_expiry_supported", + "report": "Observed: 30/30 failed requests in the configured service/window.\nThe token expired because the last release disabled automatic refresh. Every payment is affected. Revert that release and the system will recover.", + "author_note": "Proposed partly matching symptom with unsupported underlying cause; SRE assessment pending." + }, + { + "case_id": "no_evidence", + "report": "Observed: 30/30 failed requests in the configured service/window.\nNo traces or logs were supplied. The cause cannot be established from request counts alone. Next check: verify the scoped telemetry source and retrieve failed-request traces and related logs. Recovery is not established.", + "author_note": "Proposed useful uncertainty with targeted evidence request; SRE assessment pending." + }, + { + "case_id": "no_evidence", + "report": "Observed: 30/30 failed requests in the configured service/window.\nCause unknown.", + "author_note": "Proposed supported but minimally useful answer; SRE assessment pending." + }, + { + "case_id": "no_evidence", + "report": "Observed: 30/30 failed requests in the configured service/window.\nThe payment provider is experiencing a regional outage. Roll back the application and close the incident after the rollback succeeds.", + "author_note": "Proposed fabricated cause and invalid recovery advice; SRE assessment pending." + } + ] +} diff --git a/tests/core/telemetry/evaluation/calibration.py b/tests/core/telemetry/evaluation/calibration.py new file mode 100644 index 0000000000..337ee61745 --- /dev/null +++ b/tests/core/telemetry/evaluation/calibration.py @@ -0,0 +1,298 @@ +"""Prepare neutral calibration reports and compare reviewed human/judge preferences.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import random +from datetime import UTC, datetime +from itertools import combinations +from pathlib import Path +from typing import Annotated, Any, Literal + +from pydantic import Field, model_validator + +from core.telemetry.investigation import evidence_payload, sanitize +from tests.core.telemetry.evaluation.cases import Contract, load_cases +from tests.core.telemetry.evaluation.judge.score import read_runs +from tests.core.telemetry.evaluation.reporting import read_scores, write_json + + +class Example(Contract): + case_id: str + report: str = Field(min_length=1) + author_note: str = Field(min_length=1) + + +class ExampleSet(Contract): + version: Literal[1] + review_status: Literal["pending_sre_review"] + examples: list[Example] = Field(min_length=1) + + +class Rating(Contract): + candidate_id: str + case_id: str + repeat: int = Field(ge=1) + report: str + review_status: Literal["pending", "reviewed"] = "pending" + evidence_support: Annotated[int, Field(strict=True, ge=0, le=3)] | None = None + diagnostic_usefulness: Annotated[int, Field(strict=True, ge=0, le=3)] | None = None + rationale: str = "" + + @model_validator(mode="after") + def completed_rating(self) -> Rating: + if self.review_status == "reviewed" and ( + self.evidence_support is None + or self.diagnostic_usefulness is None + or not self.rationale.strip() + ): + raise ValueError("Reviewed ratings need both scores and a rationale") + return self + + +class CaseReview(Contract): + case_id: str + proposed_expectations: dict[str, Any] + payload: str + decision: Literal["pending", "accepted", "revise"] = "pending" + notes: str = "" + + +class Worksheet(Contract): + version: Literal[1] + runs_sha256: str + reviewer: str = "" + cases: list[CaseReview] + ratings: list[Rating] + + @model_validator(mode="after") + def attributable_reviews(self) -> Worksheet: + if ( + any(row.review_status == "reviewed" for row in self.ratings) + or any(case.decision != "pending" for case in self.cases) + ) and not self.reviewer.strip(): + raise ValueError("Human reviews must identify their reviewer") + return self + + +def prepare(cases_path: Path, examples_path: Path, output: Path, seed: int) -> None: + cases, dataset_hash = load_cases(cases_path) + examples_raw = examples_path.read_bytes() + examples = ExampleSet.model_validate_json(examples_raw) + sources = {case.id: case for case in cases.cases} + grouped: dict[str, list[Example]] = {} + for example in examples.examples: + if example.case_id not in sources or sources[example.case_id].split != "development": + raise ValueError("Calibration examples must reference development cases") + grouped.setdefault(example.case_id, []).append(example) + counts = {len(rows) for rows in grouped.values()} + if len(counts) != 1: + raise ValueError("Each calibration case needs the same number of candidate reports") + records: list[dict[str, Any]] = [] + ratings: list[Rating] = [] + case_reviews: list[CaseReview] = [] + rng = random.Random(seed) + seen: set[str] = set() + for case_id, candidates in sorted(grouped.items()): + source = sources[case_id] + incident = source.incident + payload = evidence_payload(incident.scope(), incident.measurement(), incident.evidence) + case_reviews.append( + CaseReview( + case_id=case_id, + payload=payload, + proposed_expectations=source.expectations.model_dump(), + ) + ) + rng.shuffle(candidates) + for repeat, example in enumerate(candidates, start=1): + report = sanitize(example.report) + candidate_id = hashlib.sha256((case_id + "\n" + report).encode()).hexdigest()[:16] + if candidate_id in seen: + raise ValueError("Duplicate calibration candidate") + seen.add(candidate_id) + prefix = f"Observed: {incident.errors:g}/{incident.total:g} failed requests in the configured service/window.\n" + records.append( + { + "kind": "result", + "case_id": case_id, + "repeat": repeat, + "candidate_id": candidate_id, + "category": source.category, + "family": source.family, + "split": source.split, + "review_status": source.review_status, + "payload": payload, + "payload_sha256": hashlib.sha256(payload.encode()).hexdigest(), + "expectations": source.expectations.model_dump(), + "report": report, + "status": "completed", + "error_type": None, + "duration_seconds": 0, + "usage": None, + "contract_checks": {"measurement_prefix_preserved": report.startswith(prefix)}, + } + ) + ratings.append( + Rating(candidate_id=candidate_id, case_id=case_id, repeat=repeat, report=report) + ) + manifest = { + "kind": "manifest", + "schema_version": 1, + "purpose": "judge_calibration", + "report_origin": "authored_examples_not_investigator_outputs", + "created_at": datetime.now(UTC).isoformat(), + "selected_cases": sorted(grouped), + "repeats": next(iter(counts)), + "seed": seed, + "dataset_sha256": dataset_hash, + "examples_sha256": hashlib.sha256(examples_raw).hexdigest(), + } + raw = ("\n".join(json.dumps(row) for row in [manifest, *records]) + "\n").encode() + worksheet = Worksheet( + version=1, runs_sha256=hashlib.sha256(raw).hexdigest(), cases=case_reviews, ratings=ratings + ) + output.parent.mkdir(parents=True, exist_ok=True) + output.mkdir(mode=0o700) # A new directory prevents accidental mixing of packs. + fd = os.open(output / "runs.jsonl", os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "wb") as stream: + stream.write(raw) + write_json(output / "review.json", worksheet.model_dump()) + + +def compare(runs_path: Path, scores_path: Path, review_path: Path) -> dict[str, Any]: + runs = read_runs(runs_path) + if runs.manifest.get("purpose") != "judge_calibration": + raise ValueError("Expected an authored calibration pack") + scores = read_scores(scores_path, runs) + review_raw = review_path.read_bytes() + worksheet = Worksheet.model_validate_json(review_raw) + if worksheet.runs_sha256 != runs.sha256: + raise ValueError("Human worksheet does not match the calibration pack") + originals = {(row["case_id"], row["repeat"]): row for row in runs.results} + reviewed: dict[tuple[str, int], Rating] = {} + for rating in worksheet.ratings: + key = rating.case_id, rating.repeat + if ( + key not in originals + or key in reviewed + or rating.report != originals[key]["report"] + or rating.candidate_id != originals[key]["candidate_id"] + ): + raise ValueError("Changed, unknown or duplicate human candidate") + reviewed[key] = rating + if reviewed.keys() != originals.keys(): + raise ValueError("Human worksheet must retain pending candidates") + source_cases = {row["case_id"]: row for row in runs.results} + case_reviews: dict[str, CaseReview] = {} + for case in worksheet.cases: + source = source_cases.get(case.case_id) + if ( + source is None + or case.case_id in case_reviews + or case.payload != source["payload"] + or case.proposed_expectations != source["expectations"] + ): + raise ValueError("Changed, unknown or duplicate case review") + case_reviews[case.case_id] = case + if case_reviews.keys() != source_cases.keys(): + raise ValueError("Missing case reviews") + metrics = { + (row["case_id"], row["repeat"], metric["name"]): metric["score"] + for row in scores.results + for metric in row["metrics"] + if metric["status"] == "completed" + } + eligible = [ + row + for row in worksheet.ratings + if row.review_status == "reviewed" and case_reviews[row.case_id].decision == "accepted" + ] + comparisons: dict[str, Any] = {} + for field, metric_name in ( + ("evidence_support", "Evidence support"), + ("diagnostic_usefulness", "Diagnostic usefulness"), + ): + stats: dict[str, Any] = { + "comparable_pairs": 0, + "agreements": 0, + "human_ties": 0, + "judge_missing_pairs": 0, + "judge_ties": 0, + "disagreements": [], + } + for left, right in combinations(eligible, 2): + if left.case_id != right.case_id: + continue + human_left, human_right = getattr(left, field), getattr(right, field) + if human_left == human_right: + stats["human_ties"] += 1 + continue + jl = metrics.get((left.case_id, left.repeat, metric_name)) + jr = metrics.get((right.case_id, right.repeat, metric_name)) + if jl is None or jr is None: + stats["judge_missing_pairs"] += 1 + continue + stats["comparable_pairs"] += 1 + if jl == jr: + stats["judge_ties"] += 1 + if (human_left - human_right) * (jl - jr) > 0: + stats["agreements"] += 1 + else: + stats["disagreements"].append( + { + "case_id": left.case_id, + "left": left.candidate_id, + "right": right.candidate_id, + "human_scores": [human_left, human_right], + "judge_scores": [jl, jr], + } + ) + stats["agreement_fraction"] = ( + stats["agreements"] / stats["comparable_pairs"] if stats["comparable_pairs"] else None + ) + comparisons[metric_name] = stats + return { + "status": "advisory_calibration_comparison", + "reviewer": worksheet.reviewer, + "runs_sha256": runs.sha256, + "scores_sha256": scores.sha256, + "review_sha256": hashlib.sha256(review_raw).hexdigest(), + "judge_model": scores.manifest["judge_model"], + "scorer_sha256": scores.manifest["scorer_sha256"], + "candidate_count": len(worksheet.ratings), + "eligible_reviewed_candidates": len(eligible), + "pending_or_unaccepted_candidates": len(worksheet.ratings) - len(eligible), + "scoring_finished": scores.finished, + "comparisons": comparisons, + "limits": [ + "Authored example ratings are not investigator accuracy.", + "Within-case pairs are correlated; inspect counts and disagreements, not only the fraction.", + "This command does not change fixture review status or validate ground truth.", + ], + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + pack = commands.add_parser("prepare", help="Offline; creates a new review directory") + pack.add_argument("--cases", type=Path, default=Path(__file__).with_name("cases.json")) + pack.add_argument("--examples", type=Path, default=Path(__file__).with_name("calibration.json")) + pack.add_argument("--seed", type=int, default=42) + pack.add_argument("--output", type=Path, required=True) + comparison = commands.add_parser("compare", help="Offline; requires explicit human ratings") + for name in ("runs", "scores", "review", "output"): + comparison.add_argument(f"--{name}", type=Path, required=True) + args = parser.parse_args() + if args.command == "prepare": + prepare(args.cases, args.examples, args.output, args.seed) + else: + write_json(args.output, compare(args.runs, args.scores, args.review)) + + +if __name__ == "__main__": + main() diff --git a/tests/core/telemetry/evaluation/reporting.py b/tests/core/telemetry/evaluation/reporting.py new file mode 100644 index 0000000000..4c4f84f44b --- /dev/null +++ b/tests/core/telemetry/evaluation/reporting.py @@ -0,0 +1,212 @@ +"""Validate saved judge checkpoints and build failure-class scorecards offline.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +from collections import Counter +from dataclasses import dataclass +from pathlib import Path +from statistics import mean +from typing import Any + +from tests.core.telemetry.evaluation.judge.score import RunSnapshot, aggregate, read_runs + + +@dataclass(frozen=True) +class ScoreSnapshot: + manifest: dict[str, Any] + results: list[dict[str, Any]] + sha256: str + incomplete_tail: bool + finished: bool + + +def read_scores(path: Path, runs: RunSnapshot) -> ScoreSnapshot: + """Bind score checkpoints to the exact generation snapshot and retain interrupted work.""" + raw = path.read_bytes() + lines = raw.splitlines(keepends=True) + events: list[dict[str, Any]] = [] + incomplete = False + for index, line in enumerate(lines): + if not line.strip(): + continue + try: + event = json.loads(line) + except (json.JSONDecodeError, UnicodeDecodeError): + if index != len(lines) - 1 or line.endswith((b"\n", b"\r")) or not events: + raise ValueError("Damaged score JSONL row") from None + incomplete = True + continue + if not isinstance(event, dict): + raise ValueError("Score events must be objects") + events.append(event) + if not events: + raise ValueError("Missing score manifest") + manifest, *events = events + if ( + manifest.get("kind") != "score_manifest" + or manifest.get("schema_version") != 1 + or manifest.get("runs_sha256") != runs.sha256 + or manifest.get("generation_manifest") != runs.manifest + ): + raise ValueError("Scores do not match the generation snapshot") + inputs = {(row["case_id"], row["repeat"]): row for row in runs.results} + checkpoints: dict[tuple[str, int], dict[str, dict[str, Any]]] = {} + complete: dict[tuple[str, int], dict[str, Any]] = {} + finished = False + for event in events: + if finished: + raise ValueError("Events follow the final summary") + if event.get("kind") == "summary": + finished = True + continue # Recompute statistics from records, never trust a stored summary. + key = event.get("case_id"), event.get("repeat") + if key not in inputs or key in complete: + raise ValueError("Unknown or duplicate score attempt") + metrics = checkpoints.setdefault(key, {}) + if event.get("kind") == "metric": + metric: dict[str, Any] = { + name: event.get(name) + for name in ("name", "status", "score", "reason", "error_type") + } + validate_metric(metric) + if metric["name"] in metrics: + raise ValueError("Duplicate metric checkpoint") + metrics[metric["name"]] = metric + elif event.get("kind") == "result": + declared = event.get("metrics", []) + for metric in declared: + validate_metric(metric) + if ( + len(declared) != len(metrics) + or {metric["name"]: metric for metric in declared} != metrics + ): + raise ValueError("Result does not match its metric checkpoints") + source = inputs[key] + if ( + event.get("generation_status") != source["status"] + or event.get("contract_checks") != source["contract_checks"] + ): + raise ValueError("Score result changed generation facts") + judge_status = event.get("judge_status") + if judge_status not in {"completed", "error", "not_run"}: + raise ValueError("Unknown judge status") + if judge_status == "completed" and ( + len(metrics) != 2 or any(metric["status"] != "completed" for metric in declared) + ): + raise ValueError("Completed judgment requires both successful metrics") + if (source["status"] == "error") != (judge_status == "not_run") or ( + judge_status == "not_run" and declared + ): + raise ValueError("Invalid skipped judgment") + complete[key] = event + else: + raise ValueError("Unknown score event") + for key, metrics in checkpoints.items(): + if key not in complete: + source = inputs[key] + complete[key] = { + "case_id": key[0], + "repeat": key[1], + "generation_status": source["status"], + "contract_checks": source["contract_checks"], + "metrics": list(metrics.values()), + "judge_status": "interrupted", + "judge_error_type": None, + } + return ScoreSnapshot( + manifest, list(complete.values()), hashlib.sha256(raw).hexdigest(), incomplete, finished + ) + + +def validate_metric(metric: dict[str, Any]) -> None: + score = metric.get("score") + if metric.get("name") not in {"Evidence support", "Diagnostic usefulness"}: + raise ValueError("Unknown metric") + if metric.get("status") == "completed": + if ( + isinstance(score, bool) + or not isinstance(score, (int, float)) + or not math.isfinite(score) + or not 0 <= score <= 1 + ): + raise ValueError("Invalid normalized metric score") + elif metric.get("status") != "error" or score is not None: + raise ValueError("Invalid failed metric") + + +def scorecard(runs: RunSnapshot, scores: ScoreSnapshot) -> dict[str, Any]: + """Show operational completion alongside advisory semantic scores.""" + if ( + runs.manifest.get("purpose") == "judge_calibration" + or runs.manifest.get("report_origin") == "authored_examples_not_investigator_outputs" + ): + raise ValueError("Authored calibration reports are not investigator baseline results") + groups: dict[str, dict[str, Any]] = {} + for category in sorted({row["category"] for row in runs.results}): + rows = [row for row in runs.results if row["category"] == category] + identities = {(row["case_id"], row["repeat"]) for row in rows} + judged = [row for row in scores.results if (row["case_id"], row["repeat"]) in identities] + groups[category] = { + "recorded_runs": len(rows), + "case_ids": sorted({row["case_id"] for row in rows}), + "generation_statuses": dict(Counter(row["status"] for row in rows)), + "duration_seconds": { + "mean": mean(row["duration_seconds"] for row in rows), + "max": max(row["duration_seconds"] for row in rows), + }, + "judgments": aggregate(judged, len(rows)), + } + expected = len(runs.manifest["selected_cases"]) * runs.manifest["repeats"] + fixture_statuses = {row["case_id"]: row["review_status"] for row in runs.results} + return { + "status": "advisory_unvalidated_baseline", + "runs_sha256": runs.sha256, + "scores_sha256": scores.sha256, + "generation_manifest": runs.manifest, + "judge_model": scores.manifest["judge_model"], + "scorer_sha256": scores.manifest["scorer_sha256"], + "expected_runs": expected, + "expected_cases": len(runs.manifest["selected_cases"]), + "recorded_cases": len(fixture_statuses), + "recorded_runs": len(runs.results), + "missing_generation_runs": expected - len(runs.results), + "missing_judge_attempts": len(runs.results) - len(scores.results), + "input_incomplete_tail": runs.incomplete_tail, + "scores_incomplete_tail": scores.incomplete_tail, + "scoring_finished": scores.finished, + "fixture_review_statuses": dict(Counter(fixture_statuses.values())), + "generation_statuses": dict(Counter(row["status"] for row in runs.results)), + "judge_statuses": dict(Counter(row["judge_status"] for row in scores.results)), + "by_failure_class": groups, + "limits": [ + "No semantic pass threshold or production accuracy claim.", + "Unsupported claims and abstention require SRE claim-level review; scores are not verified claim counts.", + ], + } + + +def write_json(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as output: + json.dump(value, output, indent=2) + output.write("\n") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--runs", type=Path, required=True) + parser.add_argument("--scores", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True, help="New JSON scorecard file") + args = parser.parse_args() + runs = read_runs(args.runs) + write_json(args.output, scorecard(runs, read_scores(args.scores, runs))) + + +if __name__ == "__main__": + main() diff --git a/tests/core/telemetry/test_calibration.py b/tests/core/telemetry/test_calibration.py new file mode 100644 index 0000000000..05a273fe0e --- /dev/null +++ b/tests/core/telemetry/test_calibration.py @@ -0,0 +1,190 @@ +"""Calibration provenance, explicit human judgments and scorecard denominators.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest +from pydantic import ValidationError + +from tests.core.telemetry.evaluation.calibration import Worksheet, compare, prepare +from tests.core.telemetry.evaluation.judge.score import read_runs, score_records +from tests.core.telemetry.evaluation.reporting import read_scores, scorecard + + +def _pack(tmp_path: Path) -> Path: + root = Path(__file__).with_name("evaluation") + destination = tmp_path / "calibration" + prepare(root / "cases.json", root / "calibration.json", destination, seed=42) + return destination + + +def _fake_scores(pack: Path) -> Path: + runs = read_runs(pack / "runs.jsonl") + + def judge(result: dict[str, Any]) -> list[dict[str, Any]]: + return [ + { + "name": name, + "status": "completed", + "score": result["repeat"] / 3, + "reason": "offline test stub", + "error_type": None, + } + for name in ("Evidence support", "Diagnostic usefulness") + ] + + events = [ + { + "kind": "score_manifest", + "schema_version": 1, + "runs_sha256": runs.sha256, + "generation_manifest": runs.manifest, + "judge_model": "offline-test-stub", + "scorer_sha256": "unit-test-only", + } + ] + score_records(runs.results, judge, events.append) + events.append({"kind": "summary"}) + path = pack / "scores.jsonl" + path.write_text("\n".join(json.dumps(event) for event in events) + "\n") + return path + + +def _review(pack: Path) -> dict[str, Any]: + worksheet: dict[str, Any] = json.loads((pack / "review.json").read_text()) + worksheet["reviewer"] = "unit-test-only" + for case in worksheet["cases"]: + case["decision"] = "accepted" + for rating in worksheet["ratings"]: + rating.update( + review_status="reviewed", + evidence_support=3 - rating["repeat"], + diagnostic_usefulness=3 - rating["repeat"], + rationale="Synthetic test rating; not an SRE judgment", + ) + return worksheet + + +def test_pack_keeps_author_notes_out_of_judge_and_review_inputs(tmp_path: Path) -> None: + pack = _pack(tmp_path) + runs = read_runs(pack / "runs.jsonl") + worksheet = Worksheet.model_validate_json((pack / "review.json").read_bytes()) + assert runs.manifest["purpose"] == "judge_calibration" + assert len(runs.results) == len(worksheet.ratings) == 9 + assert not worksheet.reviewer + assert all( + row.review_status == "pending" and row.evidence_support is None for row in worksheet.ratings + ) + assert "author_note" not in (pack / "runs.jsonl").read_text() + assert "author_note" not in (pack / "review.json").read_text() + assert all(row["candidate_id"] not in row["payload"] for row in runs.results) + with pytest.raises(FileExistsError): + _pack(tmp_path) + + +def test_pending_reviews_do_not_produce_agreement(tmp_path: Path) -> None: + pack = _pack(tmp_path) + scores = _fake_scores(pack) + result = compare(pack / "runs.jsonl", scores, pack / "review.json") + assert result["eligible_reviewed_candidates"] == 0 + assert result["pending_or_unaccepted_candidates"] == 9 + assert result["comparisons"]["Evidence support"]["agreement_fraction"] is None + with pytest.raises(ValueError, match="not investigator baseline"): + runs = read_runs(pack / "runs.jsonl") + scorecard(runs, read_scores(scores, runs)) + + +def test_pairwise_disagreements_and_human_ties_are_explicit(tmp_path: Path) -> None: + pack = _pack(tmp_path) + scores = _fake_scores(pack) + review = _review(pack) + for rating in review["ratings"]: + rating["diagnostic_usefulness"] = 2 + (pack / "review.json").write_text(json.dumps(review)) + result = compare(pack / "runs.jsonl", scores, pack / "review.json") + support = result["comparisons"]["Evidence support"] + assert support["comparable_pairs"] == len(support["disagreements"]) == 9 + assert support["agreement_fraction"] == 0 + usefulness = result["comparisons"]["Diagnostic usefulness"] + assert usefulness["human_ties"] == 9 + assert usefulness["agreement_fraction"] is None + + +@pytest.mark.parametrize( + "tampering", ["runs_hash", "report", "candidate", "case_context", "duplicate"] +) +def test_comparison_rejects_mismatched_review_material(tmp_path: Path, tampering: str) -> None: + pack = _pack(tmp_path) + scores = _fake_scores(pack) + review = _review(pack) + if tampering == "runs_hash": + review["runs_sha256"] = "other" + elif tampering == "report": + review["ratings"][0]["report"] = "changed" + elif tampering == "candidate": + review["ratings"][0]["candidate_id"] = "other" + elif tampering == "case_context": + review["cases"][0]["payload"] = "changed" + else: + review["ratings"].append(review["ratings"][0]) + (pack / "review.json").write_text(json.dumps(review)) + with pytest.raises(ValueError): + compare(pack / "runs.jsonl", scores, pack / "review.json") + + +def test_review_requires_attribution_and_complete_ratings(tmp_path: Path) -> None: + pack = _pack(tmp_path) + review = _review(pack) + review["reviewer"] = "" + with pytest.raises(ValidationError, match="reviewer"): + Worksheet.model_validate(review) + review["reviewer"] = "unit-test-only" + review["ratings"][0]["evidence_support"] = None + with pytest.raises(ValidationError, match="both scores"): + Worksheet.model_validate(review) + + +def test_scorecard_retains_interrupted_metric_and_generation_failures(tmp_path: Path) -> None: + pack = _pack(tmp_path) + generation_path = pack / "runs.jsonl" + records = [json.loads(line) for line in generation_path.read_text().splitlines()] + records[0].pop("purpose") + records[0]["report_origin"] = "offline_test_fixture" + records[1]["status"] = "partial" + generation_path.write_text("\n".join(json.dumps(row) for row in records) + "\n") + scores_path = _fake_scores(pack) + events = [json.loads(line) for line in scores_path.read_text().splitlines()] + scores_path.write_text("\n".join(json.dumps(row) for row in events[:2]) + '\n{"kind":') + runs = read_runs(generation_path) + scores = read_scores(scores_path, runs) + result = scorecard(runs, scores) + assert result["generation_statuses"]["partial"] == 1 + assert result["judge_statuses"] == {"interrupted": 1} + assert result["missing_judge_attempts"] == 8 + assert result["scores_incomplete_tail"] + assert not result["scoring_finished"] + group = result["by_failure_class"][runs.results[0]["category"]] + assert group["judgments"]["per_case_metrics"] + + +@pytest.mark.parametrize("tampering", ["hash", "duplicate_metric", "result_metrics", "nan"]) +def test_score_reader_rejects_mismatched_or_fabricated_scores( + tmp_path: Path, tampering: str +) -> None: + pack = _pack(tmp_path) + scores_path = _fake_scores(pack) + records = [json.loads(line) for line in scores_path.read_text().splitlines()] + if tampering == "hash": + records[0]["runs_sha256"] = "other" + elif tampering == "duplicate_metric": + records.insert(2, records[1]) + elif tampering == "result_metrics": + records[3]["metrics"][0]["score"] = 0 + else: + records[1]["score"] = float("nan") + scores_path.write_text("\n".join(json.dumps(row) for row in records)) + with pytest.raises(ValueError): + read_scores(scores_path, read_runs(pack / "runs.jsonl")) From 9455b48469098cf3cbc12bde1a702efdfb2bbaf4 Mon Sep 17 00:00:00 2001 From: Anwesh Nayak Date: Mon, 9 Nov 2026 15:21:32 +0000 Subject: [PATCH 2/3] test(triage): count generation failures independently of judge progress --- tests/core/telemetry/evaluation/CALIBRATION.md | 2 ++ tests/core/telemetry/evaluation/reporting.py | 12 +++++++++++- tests/core/telemetry/test_calibration.py | 3 +++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/core/telemetry/evaluation/CALIBRATION.md b/tests/core/telemetry/evaluation/CALIBRATION.md index 92b60c6671..d93bd7644f 100644 --- a/tests/core/telemetry/evaluation/CALIBRATION.md +++ b/tests/core/telemetry/evaluation/CALIBRATION.md @@ -111,6 +111,8 @@ The scorecard rejects authored calibration packs. It validates that score result match their metric checkpoints and exact generation snapshot, then groups results by failure class. It exposes generation failures, partial results, missing attempts, unfinished judging, per-case metric variation and durations. +Generation contract failures are counted from every saved investigator result, +independently of whether judging ever reaches that attempt. An interrupted score file retains completed metric checkpoints; absent metrics remain missing rather than becoming zero or perfect scores. Stored summary statistics are ignored and recomputed. Fixture review counts refer to unique diff --git a/tests/core/telemetry/evaluation/reporting.py b/tests/core/telemetry/evaluation/reporting.py index 4c4f84f44b..89979b2246 100644 --- a/tests/core/telemetry/evaluation/reporting.py +++ b/tests/core/telemetry/evaluation/reporting.py @@ -151,15 +151,22 @@ def scorecard(runs: RunSnapshot, scores: ScoreSnapshot) -> dict[str, Any]: rows = [row for row in runs.results if row["category"] == category] identities = {(row["case_id"], row["repeat"]) for row in rows} judged = [row for row in scores.results if (row["case_id"], row["repeat"]) in identities] + judgments = aggregate(judged, len(rows)) + # Generation facts use every recorded attempt, including those never judged. + judgments.pop("generation_statuses") + judgments.pop("measurement_prefix_failures") groups[category] = { "recorded_runs": len(rows), "case_ids": sorted({row["case_id"] for row in rows}), "generation_statuses": dict(Counter(row["status"] for row in rows)), + "measurement_prefix_failures": sum( + not row["contract_checks"]["measurement_prefix_preserved"] for row in rows + ), "duration_seconds": { "mean": mean(row["duration_seconds"] for row in rows), "max": max(row["duration_seconds"] for row in rows), }, - "judgments": aggregate(judged, len(rows)), + "judgments": judgments, } expected = len(runs.manifest["selected_cases"]) * runs.manifest["repeats"] fixture_statuses = {row["case_id"]: row["review_status"] for row in runs.results} @@ -181,6 +188,9 @@ def scorecard(runs: RunSnapshot, scores: ScoreSnapshot) -> dict[str, Any]: "scoring_finished": scores.finished, "fixture_review_statuses": dict(Counter(fixture_statuses.values())), "generation_statuses": dict(Counter(row["status"] for row in runs.results)), + "measurement_prefix_failures": sum( + not row["contract_checks"]["measurement_prefix_preserved"] for row in runs.results + ), "judge_statuses": dict(Counter(row["judge_status"] for row in scores.results)), "by_failure_class": groups, "limits": [ diff --git a/tests/core/telemetry/test_calibration.py b/tests/core/telemetry/test_calibration.py index 05a273fe0e..6184601c90 100644 --- a/tests/core/telemetry/test_calibration.py +++ b/tests/core/telemetry/test_calibration.py @@ -154,6 +154,7 @@ def test_scorecard_retains_interrupted_metric_and_generation_failures(tmp_path: records[0].pop("purpose") records[0]["report_origin"] = "offline_test_fixture" records[1]["status"] = "partial" + records[2]["contract_checks"]["measurement_prefix_preserved"] = False generation_path.write_text("\n".join(json.dumps(row) for row in records) + "\n") scores_path = _fake_scores(pack) events = [json.loads(line) for line in scores_path.read_text().splitlines()] @@ -167,6 +168,8 @@ def test_scorecard_retains_interrupted_metric_and_generation_failures(tmp_path: assert result["scores_incomplete_tail"] assert not result["scoring_finished"] group = result["by_failure_class"][runs.results[0]["category"]] + assert result["measurement_prefix_failures"] == group["measurement_prefix_failures"] == 1 + assert "measurement_prefix_failures" not in group["judgments"] assert group["judgments"]["per_case_metrics"] From b941db9d4adff9054e22ec5041026125be66d98e Mon Sep 17 00:00:00 2001 From: Anwesh Nayak Date: Mon, 9 Nov 2026 15:25:03 +0000 Subject: [PATCH 3/3] test(triage): expose missing generation coverage by class --- .../core/telemetry/evaluation/CALIBRATION.md | 11 ++++- tests/core/telemetry/evaluation/reporting.py | 48 ++++++++++++++++--- tests/core/telemetry/test_calibration.py | 32 +++++++++++++ 3 files changed, 83 insertions(+), 8 deletions(-) diff --git a/tests/core/telemetry/evaluation/CALIBRATION.md b/tests/core/telemetry/evaluation/CALIBRATION.md index d93bd7644f..a37b3bc1bf 100644 --- a/tests/core/telemetry/evaluation/CALIBRATION.md +++ b/tests/core/telemetry/evaluation/CALIBRATION.md @@ -109,7 +109,16 @@ uv run python -m tests.core.telemetry.evaluation.reporting \ The scorecard rejects authored calibration packs. It validates that score results match their metric checkpoints and exact generation snapshot, then groups -results by failure class. It exposes generation failures, partial results, +results by failure class. When its hash matches the generation manifest, the +included case dataset identifies selected classes even if they have zero +recorded attempts. Use optional `--cases` to supply an original custom dataset; +an explicitly supplied dataset must match the recorded hash. If the default +dataset has changed, recorded attempts still identify known classes and all +remaining selected case IDs appear in an `unclassified` coverage bucket. + +Every class includes expected, recorded and missing generation counts. No +recorded attempt means no duration or semantic score, rather than a zero-cost +success. The scorecard exposes generation failures, partial results, missing attempts, unfinished judging, per-case metric variation and durations. Generation contract failures are counted from every saved investigator result, independently of whether judging ever reaches that attempt. diff --git a/tests/core/telemetry/evaluation/reporting.py b/tests/core/telemetry/evaluation/reporting.py index 89979b2246..d240afaa15 100644 --- a/tests/core/telemetry/evaluation/reporting.py +++ b/tests/core/telemetry/evaluation/reporting.py @@ -13,6 +13,7 @@ from statistics import mean from typing import Any +from tests.core.telemetry.evaluation.cases import load_cases from tests.core.telemetry.evaluation.judge.score import RunSnapshot, aggregate, read_runs @@ -139,16 +140,40 @@ def validate_metric(metric: dict[str, Any]) -> None: raise ValueError("Invalid failed metric") -def scorecard(runs: RunSnapshot, scores: ScoreSnapshot) -> dict[str, Any]: +def scorecard( + runs: RunSnapshot, scores: ScoreSnapshot, cases_path: Path | None = None +) -> dict[str, Any]: """Show operational completion alongside advisory semantic scores.""" if ( runs.manifest.get("purpose") == "judge_calibration" or runs.manifest.get("report_origin") == "authored_examples_not_investigator_outputs" ): raise ValueError("Authored calibration reports are not investigator baseline results") + catalog: dict[str, str] = {} + for row in runs.results: + if catalog.setdefault(row["case_id"], row["category"]) != row["category"]: + raise ValueError("A case has inconsistent failure classes") + dataset, digest = load_cases(cases_path or Path(__file__).with_name("cases.json")) + verified = digest == runs.manifest.get("dataset_sha256") + if cases_path is not None and not verified: + raise ValueError("Supplied case dataset does not match generation") + if verified: + dataset_catalog = {case.id: case.category for case in dataset.cases} + if any(dataset_catalog.get(case_id) != category for case_id, category in catalog.items()): + raise ValueError("Recorded failure classes do not match the case dataset") + catalog.update(dataset_catalog) + selected = runs.manifest["selected_cases"] + unknown = sorted(case_id for case_id in selected if case_id not in catalog) + categories = {catalog[case_id] for case_id in selected if case_id in catalog} + if unknown: + categories.add("unclassified") groups: dict[str, dict[str, Any]] = {} - for category in sorted({row["category"] for row in runs.results}): - rows = [row for row in runs.results if row["category"] == category] + for category in sorted(categories): + case_ids = [ + case_id for case_id in selected if catalog.get(case_id, "unclassified") == category + ] + rows = [row for row in runs.results if row["case_id"] in case_ids] + expected_for_class = len(case_ids) * runs.manifest["repeats"] identities = {(row["case_id"], row["repeat"]) for row in rows} judged = [row for row in scores.results if (row["case_id"], row["repeat"]) in identities] judgments = aggregate(judged, len(rows)) @@ -156,15 +181,18 @@ def scorecard(runs: RunSnapshot, scores: ScoreSnapshot) -> dict[str, Any]: judgments.pop("generation_statuses") judgments.pop("measurement_prefix_failures") groups[category] = { + "expected_runs": expected_for_class, "recorded_runs": len(rows), - "case_ids": sorted({row["case_id"] for row in rows}), + "missing_generation_runs": expected_for_class - len(rows), + "case_ids": sorted(case_ids), + "recorded_case_ids": sorted({row["case_id"] for row in rows}), "generation_statuses": dict(Counter(row["status"] for row in rows)), "measurement_prefix_failures": sum( not row["contract_checks"]["measurement_prefix_preserved"] for row in rows ), "duration_seconds": { - "mean": mean(row["duration_seconds"] for row in rows), - "max": max(row["duration_seconds"] for row in rows), + "mean": mean(row["duration_seconds"] for row in rows) if rows else None, + "max": max(row["duration_seconds"] for row in rows) if rows else None, }, "judgments": judgments, } @@ -175,6 +203,9 @@ def scorecard(runs: RunSnapshot, scores: ScoreSnapshot) -> dict[str, Any]: "runs_sha256": runs.sha256, "scores_sha256": scores.sha256, "generation_manifest": runs.manifest, + "case_catalog_source": "verified_dataset" if verified else "recorded_attempts", + "case_catalog_sha256": digest if verified else None, + "unclassified_selected_cases": unknown, "judge_model": scores.manifest["judge_model"], "scorer_sha256": scores.manifest["scorer_sha256"], "expected_runs": expected, @@ -212,10 +243,13 @@ def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--runs", type=Path, required=True) parser.add_argument("--scores", type=Path, required=True) + parser.add_argument( + "--cases", type=Path, help="Optional original case dataset; hash must match generation" + ) parser.add_argument("--output", type=Path, required=True, help="New JSON scorecard file") args = parser.parse_args() runs = read_runs(args.runs) - write_json(args.output, scorecard(runs, read_scores(args.scores, runs))) + write_json(args.output, scorecard(runs, read_scores(args.scores, runs), args.cases)) if __name__ == "__main__": diff --git a/tests/core/telemetry/test_calibration.py b/tests/core/telemetry/test_calibration.py index 6184601c90..cc798a92d9 100644 --- a/tests/core/telemetry/test_calibration.py +++ b/tests/core/telemetry/test_calibration.py @@ -173,6 +173,38 @@ def test_scorecard_retains_interrupted_metric_and_generation_failures(tmp_path: assert group["judgments"]["per_case_metrics"] +@pytest.mark.parametrize("matching_catalog", [True, False]) +def test_scorecard_shows_unstarted_classes_and_missing_repeats( + tmp_path: Path, matching_catalog: bool +) -> None: + pack = _pack(tmp_path) + path = pack / "runs.jsonl" + records = [json.loads(line) for line in path.read_text().splitlines()] + records[0].pop("purpose") + records[0]["report_origin"] = "offline_test_fixture" + if not matching_catalog: + records[0]["dataset_sha256"] = "unavailable-original-dataset" + path.write_text("\n".join(json.dumps(row) for row in records[:3]) + "\n") + runs = read_runs(path) + result = scorecard(runs, read_scores(_fake_scores(pack), runs)) + assert result["missing_generation_runs"] == 7 + groups = result["by_failure_class"] + if matching_catalog: + assert groups["clear"]["expected_runs"] == 6 + assert groups["clear"]["missing_generation_runs"] == 4 + assert groups["insufficient"]["recorded_runs"] == 0 + assert groups["insufficient"]["missing_generation_runs"] == 3 + assert result["unclassified_selected_cases"] == [] + else: + assert result["unclassified_selected_cases"] == ["auth_rejection", "no_evidence"] + assert ( + groups["unclassified"]["expected_runs"] + == groups["unclassified"]["missing_generation_runs"] + == 6 + ) + assert groups["unclassified"]["duration_seconds"]["mean"] is None + + @pytest.mark.parametrize("tampering", ["hash", "duplicate_metric", "result_metrics", "nan"]) def test_score_reader_rejects_mismatched_or_fabricated_scores( tmp_path: Path, tampering: str